def test_url_data_format():
data = """
{
"rasa_nlu_data": {
"entity_synonyms": [
{
"value": "nyc",
"synonyms": ["New York City", "nyc", "the big apple"]
}
],
"common_examples" : [
{
"text": "show me flights to New York City",
"intent": "unk",
"entities": [
{
"entity": "destination",
"start": 19,
"end": 32,
"value": "NYC"
}
]
}
]
}
}"""
fname = io_utils.create_temporary_file(data.encode("utf-8"),
suffix="_tmp_training_data.json",
mode="w+b")
data = io_utils.read_json_file(fname)
assert data is not None
validate_rasa_nlu_data(data)
def _is_nlu_file(file_path: Text) -> bool:
with open(file_path, encoding="utf-8") as f:
if file_path.endswith(".json"):
content = io_utils.read_json_file(file_path)
is_nlu_file = (isinstance(content, dict)
and content.get("rasa_nlu_data") is not None)
else:
is_nlu_file = any(_contains_nlu_pattern(l) for l in f)
return is_nlu_file
def test_core_models_in_directory(model_directory: Text, stories: Text, output: Text):
from rasa.core.test import compare_models_in_dir, plot_core_results
loop = asyncio.get_event_loop()
loop.run_until_complete(compare_models_in_dir(model_directory, stories, output))
story_n_path = os.path.join(model_directory, NUMBER_OF_TRAINING_STORIES_FILE)
number_of_stories = io_utils.read_json_file(story_n_path)
plot_core_results(output, number_of_stories)
def generate_domain_file(conf: BotsConf):
cnt = io_utils.read_yaml_file(f'{conf.templates_dir}/domain.yml')
intents = cnt['intents']
actions = cnt['actions']
for f in glob.glob(conf.ruleset_files):
rules = io_utils.read_json_file(f)
for rule in rules:
intents.append({rule['intent']: {'triggers': rule['action']}})
actions.append(rule['action'])
return cnt
def test_whitespace_does_not_throw_error():
import rasa.utils.io as io_utils
texts = io_utils.read_json_file(
"data/test_tokenizers/naughty_strings.json")
tk = WhitespaceTokenizer()
for text in texts:
tk.tokenize(Message(text), attribute=TEXT)
def load(cls,
component_meta=None,
model_dir=None,
model_metadata=None,
cached_component=None,
**kwargs):
file_name = component_meta.get("composite_entities_file",
COMPOSITE_ENTITIES_FILE_NAME)
composite_entities_file = os.path.join(model_dir, file_name)
if os.path.isfile(composite_entities_file):
composite_entities = read_json_file(composite_entities_file)
else:
composite_entities = []
warnings.warn("Failed to load composite entities"
'file from "{}"'.format(composite_entities_file))
return cls(component_meta, composite_entities)
def plot_curve(
output_directory: Text,
number_of_examples: List[int],
x_label_text: Text,
y_label_text: Text,
graph_path: Text,
) -> None:
"""Plot the results from a model comparison.
Args:
output_directory: Output directory to save resulting plots to
number_of_examples: Number of examples per run
x_label_text: text for the x axis
y_label_text: text for the y axis
graph_path: output path of the plot
"""
import matplotlib.pyplot as plt
ax = plt.gca()
# load results from file
data = io_utils.read_json_file(os.path.join(output_directory,
RESULTS_FILE))
x = number_of_examples
# compute mean of all the runs for different configs
for label in data.keys():
if len(data[label]) == 0:
continue
mean = np.mean(data[label], axis=0)
std = np.std(data[label], axis=0)
ax.plot(x, mean, label=label, marker=".")
ax.fill_between(
x,
[m - s for m, s in zip(mean, std)],
[m + s for m, s in zip(mean, std)],
color="#6b2def",
alpha=0.2,
)
ax.legend(loc=4)
ax.set_xlabel(x_label_text)
ax.set_ylabel(y_label_text)
plt.savefig(graph_path, format="pdf")
logger.info(f"Comparison graph saved to '{graph_path}'.")
def load(
cls,
meta: Dict[Text, Any],
model_dir: Optional[Text] = None,
model_metadata: Optional[Metadata] = None,
cached_component: Optional["RegexEntityExtractor"] = None,
**kwargs: Any,
) -> "RegexEntityExtractor":
file_name = meta.get("file")
regex_file = os.path.join(model_dir, file_name)
if os.path.exists(regex_file):
patterns = io_utils.read_json_file(regex_file)
return RegexEntityExtractor(meta, patterns=patterns)
return RegexEntityExtractor(meta)
def _read_composite_entities(self):
"""Read the defined composite patterns from the train file. We have
to manually load the file, as rasa strips our custom information.
"""
try:
files = [self.component_config[COMPOSITE_PATTERNS_PATH]]
except:
warnings.warn(
"No composite entity patterns path set in config.yml")
try:
files = self._get_train_files_cmd()
except:
warnings.warn("No train file specified in cli command.")
try:
files = self._get_train_files_http()
except:
warnings.warn(
"The CompositeEntityExtractor could not load "
"the train file.")
return []
composite_entities = []
for file in files:
file_content = read_json_file(file)
if "rasa_nlu_data" in file_content:
try:
rasa_nlu_data = file_content.get("rasa_nlu_data")
composite_entities_in_file = rasa_nlu_data[
"composite_entities"]
except KeyError:
pass
else:
composite_entities.extend(composite_entities_in_file)
elif "composite_entities" in file_content:
try:
composite_entities_in_file = file_content.get(
"composite_entities")
except KeyError:
pass
else:
composite_entities.extend(composite_entities_in_file)
if not composite_entities:
warnings.warn(
"CompositeEntityExtractor was added to the "
"pipeline but no composite entities have been defined.")
return composite_entities
def test_write_json_file(tmp_path: Path):
expected = {"abc": "dasds", "list": [1, 2, 3, 4], "nested": {"a": "b"}}
file_path = str(tmp_path / "abc.txt")
io_utils.dump_obj_as_json_to_file(file_path, expected)
assert io_utils.read_json_file(file_path) == expected
def test_example_training_data_is_valid():
demo_json = "data/examples/rasa/demo-rasa.json"
data = io_utils.read_json_file(demo_json)
validate_rasa_nlu_data(data)
