def evaluate(self, data: Text, project: Optional[Text] = None, model: Optional[Text] = None) -> Dict[Text, Any]: """Perform a model evaluation.""" project = project or RasaNLUModelConfig.DEFAULT_PROJECT_NAME model = model or None file_name = utils.create_temporary_file(data, "_training_data") if project not in self.project_store: raise InvalidProjectError("Project {} could not " "be found".format(project)) model_name = self.project_store[project]._dynamic_load_model(model) self.project_store[project]._loader_lock.acquire() try: if not self.project_store[project]._models.get(model_name): interpreter = self.project_store[project]. \ _interpreter_for_model(model_name) self.project_store[project]._models[model_name] = interpreter finally: self.project_store[project]._loader_lock.release() return run_evaluation( data_path=file_name, model=self.project_store[project]._models[model_name], errors_filename=None )
def test_url_data_format(): data = u""" { "rasa_nlu_data": { "entity_synonyms": [ { "value": "nyc", "synonyms": ["New York City", "nyc", "the big apple"] } ], "common_examples" : [ { "text": "show me flights to New York City", "intent": "unk", "entities": [ { "entity": "destination", "start": 19, "end": 32, "value": "NYC" } ] } ] } }""" fname = utils.create_temporary_file(data.encode("utf-8"), suffix="_tmp_training_data.json", mode="w+b") data = utils.read_json_file(fname) assert data is not None validate_rasa_nlu_data(data)
def dump_to_data_file(data): if isinstance(data, six.string_types): data_string = data else: data_string = utils.json_to_string(data) return utils.create_temporary_file(data_string, "_training_data")
def evaluate(self, data, project=None, model=None): # type: (Text, Optional[Text], Optional[Text]) -> Dict[Text, Any] """Perform a model evaluation.""" project = project or RasaNLUModelConfig.DEFAULT_PROJECT_NAME model = model or None file_name = utils.create_temporary_file(data, "_training_data") if project not in self.project_store: raise InvalidProjectError("Project {} could not " "be found".format(project)) model_name = self.project_store[project]._dynamic_load_model(model) self.project_store[project]._loader_lock.acquire() try: if not self.project_store[project]._models.get(model_name): interpreter = self.project_store[project]. \ _interpreter_for_model(model_name) self.project_store[project]._models[model_name] = interpreter finally: self.project_store[project]._loader_lock.release() return run_evaluation( data_path=file_name, model=self.project_store[project]._models[model_name], errors_filename=None)
def test_emojis_in_tmp_file(): test_data = """ data: - one ππ― π©πΏβπ»π¨πΏβπ» - two Β£ (?u)\\b\\w+\\b f\u00fcr """ test_file = utils.create_temporary_file(test_data) with io.open(test_file, mode='r', encoding="utf-8") as f: content = f.read() actual = utils.read_yaml(content) assert actual["data"][0] == "one ππ― π©πΏβπ»π¨πΏβπ»" assert actual["data"][1] == "two Β£ (?u)\\b\\w+\\b fΓΌr"
def test_emojis_in_tmp_file(): test_data = """ data: - one π - two Β£ """ test_file = utils.create_temporary_file(test_data) with io.open(test_file, mode='r', encoding="utf-8") as f: content = f.read() actual = utils.read_yaml(content) assert actual["data"][0] == "one π" assert actual["data"][1] == "two Β£"
def load_data_from_url(url, language='en'): # type: (Text, Optional[Text]) -> TrainingData """Load training data from a URL.""" if not utils.is_url(url): raise requests.exceptions.InvalidURL(url) try: response = requests.get(url) response.raise_for_status() temp_data_file = utils.create_temporary_file(response.content) return _load(temp_data_file, language) except Exception as e: logger.warning("Could not retrieve training data " "from URL:\n{}".format(e))
def download_file_from_url(url: Text) -> Text: """Download a story file from a url and persists it into a temp file. Returns the file path of the temp file that contains the downloaded content.""" if not nlu_utils.is_url(url): raise InvalidURL(url) response = requests.get(url) response.raise_for_status() filename = nlu_utils.create_temporary_file(response.content, mode="w+b") return filename
def load_data_from_endpoint(data_endpoint, language='en'): # type: (EndpointConfig, Optional[Text]) -> TrainingData """Load training data from a URL.""" if not utils.is_url(data_endpoint.url): raise requests.exceptions.InvalidURL(data_endpoint.url) try: response = data_endpoint.request("get") response.raise_for_status() temp_data_file = utils.create_temporary_file(response.content, mode="w+b") return _load(temp_data_file, language) except Exception as e: logger.warning("Could not retrieve training data " "from URL:\n{}".format(e))
async def download_file_from_url(url: Text) -> Text: """Download a story file from a url and persists it into a temp file. Returns the file path of the temp file that contains the downloaded content.""" from rasa_nlu import utils as nlu_utils if not nlu_utils.is_url(url): raise InvalidURL(url) async with aiohttp.ClientSession() as session: async with session.get(url, raise_for_status=True) as resp: filename = nlu_utils.create_temporary_file(await resp.read(), mode="w+b") return filename
def download_file_from_url(url): # type: (Text) -> Text """Download a story file from a url and persists it into a temp file. Returns the file path of the temp file that contains the downloaded content.""" if not nlu_utils.is_url(url): raise InvalidURL(url) response = requests.get(url) response.raise_for_status() filename = nlu_utils.create_temporary_file(response.content, mode="w+b") return filename
def load_data_from_endpoint(data_endpoint: EndpointConfig, language: Optional[Text] = 'en') -> 'TrainingData': """Load training data from a URL.""" if not utils.is_url(data_endpoint.url): raise requests.exceptions.InvalidURL(data_endpoint.url) try: response = data_endpoint.request("get") response.raise_for_status() temp_data_file = utils.create_temporary_file(response.content, mode="w+b") training_data = _load(temp_data_file, language) return training_data except Exception as e: logger.warning("Could not retrieve training data " "from URL:\n{}".format(e))
def evaluate(self, data, project=None, model=None): # type: (Text, Optional[Text], Optional[Text]) -> Dict[Text, Any] """Perform a model evaluation.""" project = project or RasaNLUModelConfig.DEFAULT_PROJECT_NAME model = model or None file_name = utils.create_temporary_file(data, "_training_data") test_data = load_data(file_name) if project not in self.project_store: raise InvalidProjectError("Project {} could not " "be found".format(project)) preds_json = self.parse_training_examples(test_data.intent_examples, project, model) predictions = [ {"text": e.text, "intent": e.data.get("intent"), "predicted": p.get("intent", {}).get("name"), "confidence": p.get("intent", {}).get("confidence")} for e, p in zip(test_data.intent_examples, preds_json) ] y_true = [e.data.get("intent") for e in test_data.intent_examples] y_true = clean_intent_labels(y_true) y_pred = [p.get("intent", {}).get("name") for p in preds_json] y_pred = clean_intent_labels(y_pred) report, precision, f1, accuracy = get_evaluation_metrics(y_true, y_pred) return { "intent_evaluation": { "report": report, "predictions": predictions, "precision": precision, "f1_score": f1, "accuracy": accuracy} }