def load_rasa_data(filename):
# type: (Text) -> TrainingData
"""Loads training data stored in the rasa NLU data format."""
with io.open(filename, encoding="utf-8-sig") as f:
data = json.loads(f.read())
validate_rasa_nlu_data(data)
common = data['rasa_nlu_data'].get("common_examples", list())
intent = data['rasa_nlu_data'].get("intent_examples", list())
entity = data['rasa_nlu_data'].get("entity_examples", list())
return TrainingData(intent, entity, common)
def test_train_with_empty_data(component_builder):
_config = utilities.base_test_conf("all_components")
trainer = Trainer(_config, component_builder)
trainer.train(TrainingData())
persistor = create_persistor(_config)
persisted_path = trainer.persist(_config['path'],
persistor,
model_name=_config['name'])
loaded = utilities.load_interpreter_for_model(_config, persisted_path,
component_builder)
assert loaded.pipeline
assert loaded.parse("hello", time=None) is not None
def _read_intent(self, intent_js, examples_js):
"""Reads the intent and examples from respective jsons."""
from rasa_nlu.training_data import Message, TrainingData
intent = intent_js.get("name")
training_examples = []
for ex in examples_js:
text, entities = self._join_text_chunks(ex['data'])
training_examples.append(Message.build(text, intent, entities))
return TrainingData(training_examples)
def test_train_with_empty_data(language, pipeline, component_builder, tmpdir):
_config = RasaNLUModelConfig({"pipeline": pipeline, "language": language})
trainer = Trainer(_config, component_builder)
trainer.train(TrainingData())
persistor = create_persistor(_config)
persisted_path = trainer.persist(tmpdir.strpath,
persistor,
project_name="my_project")
loaded = Interpreter.load(persisted_path, component_builder)
assert loaded.pipeline
assert loaded.parse("hello") is not None
assert loaded.parse("Hello today is Monday, again!") is not None
def train(cfg_name, model_name):
from rasa_nlu.train import create_trainer
from rasa_nlu.train import create_persistor
from rasa_nlu.training_data import TrainingData
config = RasaNLUConfig(cfg_name)
trainer = create_trainer(config)
persistor = create_persistor(config)
training_data = TrainingData(config['data'], config['backend'], nlp=trainer.nlp)
trainer.train(training_data)
trainer.persist(os.path.join("test_models", model_name), persistor, create_unique_subfolder=False)
def load_luis_data(filename, tokenizer):
# type: (str, Optional[Tokenizer]) -> TrainingData
"""Loads training data stored in the LUIS.ai data format."""
warnings.warn(
"""LUIS data may not always be correctly imported because entity locations are specified by tokens.
If you use a tokenizer which behaves differently from LUIS's your entities might not be correct"""
)
if not tokenizer:
raise ValueError(
"Can not load luis data without a specified tokenizer " +
"(e.g. using the configuration value `luis_data_tokenizer`)")
intent_examples = []
entity_examples = []
common_examples = []
with io.open(filename, encoding="utf-8-sig") as f:
data = json.loads(f.read())
for s in data["utterances"]:
text = s.get("text")
tokens = [t for t in tokenizer.tokenize(text)]
intent = s.get("intent")
entities = []
for e in s.get("entities") or []:
i, ii = e["startPos"], e["endPos"] + 1
_regex = u"\s*".join([re.escape(s) for s in tokens[i:ii]])
expr = re.compile(_regex)
m = expr.search(text)
start, end = m.start(), m.end()
val = text[start:end]
entities.append({
"entity": e["entity"],
"value": val,
"start": start,
"end": end
})
if intent and entities:
common_examples.append({
"text": text,
"intent": intent,
"entities": entities
})
elif intent:
intent_examples.append({"text": text, "intent": intent})
elif entities:
entity_examples.append({
"text": text,
"intent": intent,
"entities": entities
})
return TrainingData(intent_examples, entity_examples, common_examples)
def _write_nlu_to_file(export_nlu_path: Text, evts: List[Dict[Text,
Any]]) -> None:
"""Write the nlu data of the sender_id to the file paths."""
from rasa_nlu.training_data import TrainingData
msgs = _collect_messages(evts)
# noinspection PyBroadException
try:
previous_examples = load_data(export_nlu_path)
except Exception as e:
logger.exception("An exception occurred while trying to load the "
"NLU data.")
export_nlu_path = questionary.text(
message="Could not load existing NLU data, please "
"specify where to store NLU data learned in "
"this session (this will overwrite any "
"existing file). {}".format(str(e)),
default=PATHS["backup"]).ask()
if export_nlu_path is None:
return
previous_examples = TrainingData()
nlu_data = previous_examples.merge(TrainingData(msgs))
# need to guess the format of the file before opening it to avoid a read
# in a write
if _guess_format(export_nlu_path) in {"md", "unk"}:
fformat = "md"
else:
fformat = "json"
with open(export_nlu_path, 'w', encoding="utf-8") as f:
if fformat == "md":
f.write(nlu_data.as_markdown())
else:
f.write(nlu_data.as_json())
def load_data(resource_name, language='en'):
# type: (Text, Optional[Text]) -> TrainingData
"""Load training data from disk. Merges them if multiple files are found."""
files = utils.list_files(resource_name)
data_sets = [_load(f, language) for f in files]
data_sets = [ds for ds in data_sets if ds]
if len(data_sets) == 0:
return TrainingData()
elif len(data_sets) == 1:
return data_sets[0]
else:
return data_sets[0].merge(*data_sets[1:])
def reads(self, s, **kwargs):
"""Read markdown string and create TrainingData object"""
self.__init__()
s = self._strip_comments(s)
for line in s.splitlines():
line = line.strip()
header = self._find_section_header(line)
if header:
self._set_current_section(header[0], header[1])
else:
self._parse_item(line)
return TrainingData(self.training_examples, self.entity_synonyms, self.regex_features)
def test_count_vector_featurizer(sentence, expected):
from rasa_nlu.featurizers.count_vectors_featurizer import \
CountVectorsFeaturizer
ftr = CountVectorsFeaturizer({"token_pattern": r'(?u)\b\w+\b'})
message = Message(sentence)
message.set("intent", "bla")
data = TrainingData([message])
ftr.train(data)
ftr.process(message)
assert np.all(message.get("text_features")[0] == expected)
def load_markdown_data(filenames):
# type: (List[Text]) -> TrainingData
"""Loads training data stored in markdown data format."""
from rasa_nlu.utils.md_to_json import MarkdownToJson
common_examples = list()
known_synonyms = {}
for filename in filenames:
data = MarkdownToJson(filename)
common_examples += data.common_examples
known_synonyms = get_entity_synonyms_dict(data.entity_synonyms,
known_synonyms)
return TrainingData(common_examples, known_synonyms)
def load_data(resource_name, language='en'):
# type: (Text, Optional[Text]) -> TrainingData
"""Loads training data from disk and merges them if multiple files are found."""
files = utils.recursively_find_files(resource_name)
data_sets = [_load(f, language) for f in files]
# Dialogflow has files that we don't read directly, these return None
data_sets = [ds for ds in data_sets if ds]
if len(data_sets) == 0:
return TrainingData()
elif len(data_sets) == 1:
return data_sets[0]
else:
return data_sets[0].merge(*data_sets[1:])
def load_dialogflow_data(files, language):
# type: (List[Text]) -> TrainingData
"""Loads training data stored in the Dialogflow data format."""
training_examples = []
entity_synonyms = {}
for filename in files:
data = _read_json_from_file(filename)
# Language specific extensions
usersays_file_ext = '_usersays_{}.json'.format(language)
synonyms_file_ext = '_entries_{}.json'.format(language)
if filename.endswith(usersays_file_ext):
synonyms_filename = filename.replace(usersays_file_ext, '.json')
root_f_data = _read_json_from_file(synonyms_filename)
intent = root_f_data.get("name")
for s in data:
text = "".join([chunk["text"] for chunk in s.get("data")])
# add entities to each token, if available
entities = []
for e in [chunk
for chunk in s.get("data")
if "alias" in chunk or "meta" in chunk]:
start = text.find(e["text"])
end = start + len(e["text"])
val = text[start:end]
entity_type = e["alias"] if "alias" in e else e["meta"]
if entity_type != u'@sys.ignore':
entities.append(
{
"entity": entity_type,
"value": val,
"start": start,
"end": end
}
)
data = {}
if intent:
data["intent"] = intent
if entities is not None:
data["entities"] = entities
training_examples.append(Message(text, data))
elif filename.endswith(synonyms_file_ext):
# create synonyms dictionary
for entry in data:
if "value" in entry and "synonyms" in entry:
for synonym in entry["synonyms"]:
entity_synonyms[synonym] = entry["value"]
return TrainingData(training_examples, entity_synonyms)
def test_count_vector_featurizer(sentence, expected):
from rasa_nlu.featurizers.count_vectors_featurizer import \
CountVectorsFeaturizer
ftr = CountVectorsFeaturizer({"token_pattern": r'(?u)\b\w+\b'})
train_message = Message(sentence)
# this is needed for a valid training example
train_message.set("intent", "bla")
data = TrainingData([train_message])
ftr.train(data)
test_message = Message(sentence)
ftr.process(test_message)
assert np.all(test_message.get("text_features") == expected)
def _write_nlu_to_file(export_nlu_path, evts):
# type: (Text, List[Dict[Text, Any]]) -> None
"""Write the nlu data of the sender_id to the file paths."""
msgs = _collect_messages(evts)
# noinspection PyBroadException
try:
previous_examples = load_data(export_nlu_path)
except Exception:
questions = [{
"name":
"export nlu",
"type":
"input",
"message":
"Could not load existing NLU data, please "
"specify where to store NLU data learned in "
"this session (this will overwrite any "
"existing file)",
"default":
PATHS["backup"]
}]
answers = prompt(questions)
export_nlu_path = answers["export nlu"]
previous_examples = TrainingData()
nlu_data = previous_examples.merge(TrainingData(msgs))
with io.open(export_nlu_path, 'w', encoding="utf-8") as f:
if _guess_format(export_nlu_path) in ["md", "unk"]:
f.write(nlu_data.as_markdown())
else:
f.write(nlu_data.as_json())
def load_luis_data(filename):
# type: (Text) -> TrainingData
"""Loads training data stored in the LUIS.ai data format."""
intent_examples = []
entity_examples = []
common_examples = []
with io.open(filename, encoding="utf-8-sig") as f:
data = json.loads(f.read())
# Simple check to ensure we support this luis data schema version
if not data["luis_schema_version"].startswith("2"):
raise Exception(
"Invalid luis data schema version {}, should be 2.x.x. ".format(
data["luis_schema_version"]) +
"Make sure to use the latest luis version (e.g. by downloading your data again)."
)
for s in data["utterances"]:
text = s.get("text")
intent = s.get("intent")
entities = []
for e in s.get("entities") or []:
start, end = e["startPos"], e["endPos"] + 1
val = text[start:end]
entities.append({
"entity": e["entity"],
"value": val,
"start": start,
"end": end
})
if intent and entities:
common_examples.append({
"text": text,
"intent": intent,
"entities": entities
})
elif intent:
intent_examples.append({"text": text, "intent": intent})
elif entities:
entity_examples.append({
"text": text,
"intent": intent,
"entities": entities
})
return TrainingData(intent_examples, entity_examples, common_examples)
def test_unintentional_synonyms_capitalized(component_builder):
_config = utilities.base_test_conf("all_components")
ner_syn = component_builder.create_component("ner_synonyms", _config)
examples = [
Message("Any Mexican restaurant will do", {
"intent": "restaurant_search",
"entities": [{"start": 4, "end": 11, "value": "Mexican", "entity": "cuisine"}]
}),
Message("I want Tacos!", {
"intent": "restaurant_search",
"entities": [{"start": 7, "end": 12, "value": "Mexican", "entity": "cuisine"}]
})
]
ner_syn.train(TrainingData(training_examples=examples), _config)
assert ner_syn.synonyms.get("mexican") is None
assert ner_syn.synonyms.get("tacos") == "Mexican"
def reads(self, s: Text, **kwargs: Any) -> 'TrainingData':
"""Read markdown string and create TrainingData object"""
from rasa_nlu.training_data import TrainingData
self.__init__()
s = self._strip_comments(s)
for line in s.splitlines():
line = line.strip()
header = self._find_section_header(line)
if header:
self._set_current_section(header[0], header[1])
else:
self._parse_item(line)
self._load_files(line)
return TrainingData(self.training_examples, self.entity_synonyms,
self.regex_features, self.lookup_tables)
def load_api_data(files):
# type: (List[Text]) -> TrainingData
"""Loads training data stored in the API.ai data format."""
training_examples = []
entity_synonyms = {}
for filename in files:
with io.open(filename, encoding="utf-8-sig") as f:
data = json.loads(f.read())
# get only intents, skip the rest. The property name is the target class
if "userSays" in data:
intent = data.get("name")
for s in data["userSays"]:
text = "".join([chunk["text"] for chunk in s.get("data")])
# add entities to each token, if available
entities = []
for e in [
chunk for chunk in s.get("data")
if "alias" in chunk or "meta" in chunk
]:
start = text.find(e["text"])
end = start + len(e["text"])
val = text[start:end]
entities.append({
"entity":
e["alias"] if "alias" in e else e["meta"],
"value":
val,
"start":
start,
"end":
end
})
data = {"text": text}
if intent:
data["intent"] = intent
if entities:
data["entities"] = entities
training_examples.append(data)
# create synonyms dictionary
if "name" in data and "entries" in data:
for entry in data["entries"]:
if "value" in entry and "synonyms" in entry:
for synonym in entry["synonyms"]:
entity_synonyms[synonym] = entry["value"]
return TrainingData(training_examples, entity_synonyms)
def test_crf_extractor(spacy_nlp):
from rasa_nlu.extractors.crf_entity_extractor import CRFEntityExtractor
ext = CRFEntityExtractor()
examples = [
Message(
"anywhere in the west", {
"intent":
"restaurant_search",
"entities": [{
"start": 16,
"end": 20,
"value": "west",
"entity": "location"
}],
"spacy_doc":
spacy_nlp("anywhere in the west")
}),
Message(
"central indian restaurant", {
"intent":
"restaurant_search",
"entities": [{
"start": 0,
"end": 7,
"value": "central",
"entity": "location"
}],
"spacy_doc":
spacy_nlp("central indian restaurant")
})
]
config = {
"entity_crf_BILOU_flag": True,
"entity_crf_features": ext.crf_features
}
ext.train(TrainingData(training_examples=examples), config)
sentence = 'anywhere in the west'
crf_format = ext._from_text_to_crf(
Message(sentence, {"spacy_doc": spacy_nlp(sentence)}))
assert ([word[0]
for word in crf_format] == ['anywhere', 'in', 'the', 'west'])
feats = ext._sentence_to_features(crf_format)
assert ('BOS' in feats[0])
assert ('EOS' in feats[-1])
assert ('0:low:in' in feats[1])
sentence = 'anywhere in the west'
ext.extract_entities(Message(sentence, {"spacy_doc": spacy_nlp(sentence)}))
def test_crf_extractor(spacy_nlp, ner_crf_pos_feature_config):
from rasa_nlu.extractors.crf_entity_extractor import CRFEntityExtractor
ext = CRFEntityExtractor(component_config=ner_crf_pos_feature_config)
examples = [
Message("anywhere in the west", {
"intent": "restaurant_search",
"entities": [{"start": 16, "end": 20,
"value": "west", "entity": "location"}],
"spacy_doc": spacy_nlp("anywhere in the west")
}),
Message("central indian restaurant", {
"intent": "restaurant_search",
"entities": [
{"start": 0, "end": 7, "value": "central",
"entity": "location", "extractor": "random_extractor"},
{"start": 8, "end": 14, "value": "indian",
"entity": "cuisine", "extractor": "ner_crf"}
],
"spacy_doc": spacy_nlp("central indian restaurant")
})]
# uses BILOU and the default features
ext.train(TrainingData(training_examples=examples), RasaNLUModelConfig())
sentence = 'anywhere in the west'
doc = {"spacy_doc": spacy_nlp(sentence)}
crf_format = ext._from_text_to_crf(Message(sentence, doc))
assert [word[0] for word in crf_format] == ['anywhere', 'in', 'the', 'west']
feats = ext._sentence_to_features(crf_format)
assert 'BOS' in feats[0]
assert 'EOS' in feats[-1]
assert feats[1]['0:low'] == "in"
sentence = 'anywhere in the west'
ext.extract_entities(Message(sentence, {"spacy_doc": spacy_nlp(sentence)}))
filtered = ext.filter_trainable_entities(examples)
assert filtered[0].get('entities') == [
{"start": 16, "end": 20, "value": "west", "entity": "location"}
], 'Entity without extractor remains'
assert filtered[1].get('entities') == [
{"start": 8, "end": 14,
"value": "indian", "entity": "cuisine", "extractor": "ner_crf"}
], 'Only ner_crf entity annotation remains'
assert examples[1].get('entities')[0] == {
"start": 0, "end": 7,
"value": "central", "entity": "location",
"extractor": "random_extractor"
}, 'Original examples are not mutated'
def test_count_vector_featurizer(sentence, expected):
from rasa_nlu.featurizers.count_vectors_featurizer import \
CountVectorsFeaturizer
ftr = CountVectorsFeaturizer({"min_ngram": 1,
"max_ngram": 2,
"analyzer": 'char'})
train_message = Message(sentence)
# this is needed for a valid training example
train_message.set("intent", "bla")
data = TrainingData([train_message])
ftr.train(data)
test_message = Message(sentence)
ftr.process(test_message)
assert np.all(test_message.get("text_features") == expected)
def test_count_vector_featurizer_oov_token(sentence, expected):
from rasa_nlu.featurizers.count_vectors_featurizer import \
CountVectorsFeaturizer
ftr = CountVectorsFeaturizer({
"token_pattern": r'(?u)\b\w+\b',
"OOV_token": '__oov__'
})
train_message = Message(sentence)
train_message.set("intent", "bla")
data = TrainingData([train_message])
ftr.train(data)
test_message = Message(sentence)
ftr.process(test_message)
assert np.all(test_message.get("text_features") == expected)
def load_luis_data(filename):
# type: (Text) -> TrainingData
"""Loads training data stored in the LUIS.ai data format."""
training_examples = []
regex_features = []
with io.open(filename, encoding="utf-8-sig") as f:
data = json.loads(f.read())
# Simple check to ensure we support this luis data schema version
if not data["luis_schema_version"].startswith("2"):
raise Exception(
"Invalid luis data schema version {}, should be 2.x.x. ".format(
data["luis_schema_version"]) +
"Make sure to use the latest luis version (e.g. by downloading your data again)."
)
for r in data.get("regex_features", []):
if r.get("activated", False):
regex_features.append({
"name": r.get("name"),
"pattern": r.get("pattern")
})
for s in data["utterances"]:
text = s.get("text")
intent = s.get("intent")
entities = []
for e in s.get("entities") or []:
start, end = e["startPos"], e["endPos"] + 1
val = text[start:end]
entities.append({
"entity": e["entity"],
"value": val,
"start": start,
"end": end
})
data = {"entities": entities}
if intent:
data["intent"] = intent
training_examples.append(Message(text, data))
return TrainingData(training_examples, regex_features=regex_features)
def load_data(resource_name: Text,
language: Optional[Text] = 'en') -> 'TrainingData':
"""Load training data from disk.
Merges them if loaded from disk and multiple files are found."""
from rasa_nlu.training_data import TrainingData
files = utils.list_files(resource_name)
data_sets = [_load(f, language) for f in files]
data_sets = [ds for ds in data_sets if ds]
if len(data_sets) == 0:
training_data = TrainingData()
elif len(data_sets) == 1:
training_data = data_sets[0]
else:
training_data = data_sets[0].merge(*data_sets[1:])
training_data.validate()
return training_data
def read_from_json(self, js: Dict[Text, Any],
**kwargs: Any) -> 'TrainingData':
"""Loads training data stored in the LUIS.ai data format."""
from rasa_nlu.training_data import Message, TrainingData
training_examples = []
regex_features = []
# Simple check to ensure we support this luis data schema version
if not js["luis_schema_version"].startswith("2"):
raise Exception("Invalid luis data schema version {}, "
"should be 2.x.x. "
"Make sure to use the latest luis version "
"(e.g. by downloading your data again)."
"".format(js["luis_schema_version"]))
for r in js.get("regex_features", []):
if r.get("activated", False):
regex_features.append({
"name": r.get("name"),
"pattern": r.get("pattern")
})
for s in js["utterances"]:
text = s.get("text")
intent = s.get("intent")
entities = []
for e in s.get("entities") or []:
start, end = e["startPos"], e["endPos"] + 1
val = text[start:end]
entities.append({
"entity": e["entity"],
"value": val,
"start": start,
"end": end
})
data = {"entities": entities}
if intent:
data["intent"] = intent
training_examples.append(Message(text, data))
return TrainingData(training_examples, regex_features=regex_features)
def load_wit_data(filename):
# type: (Text) -> TrainingData
"""Loads training data stored in the WIT.ai data format."""
intent_examples = []
entity_examples = []
common_examples = []
with io.open(filename, encoding="utf-8-sig") as f:
data = json.loads(f.read())
for s in data["data"]:
entities = s.get("entities")
if entities is None:
continue
text = s.get("text")
intents = [e["value"] for e in entities if e["entity"] == 'intent']
intent = intents[0].strip("\"") if intents else None
entities = [
e for e in entities
if ("start" in e and "end" in e and e["entity"] != 'intent')
]
for e in entities:
e["value"] = e["value"].strip(
"\""
) # for some reason wit adds additional quotes around entity values
if intent and entities:
common_examples.append({
"text": text,
"intent": intent,
"entities": entities
})
elif intent:
intent_examples.append({"text": text, "intent": intent})
elif entities:
entity_examples.append({
"text": text,
"intent": intent,
"entities": entities
})
return TrainingData(intent_examples, entity_examples, common_examples)
def test_count_vector_featurizer_using_tokens(tokens, expected):
from rasa_nlu.featurizers.count_vectors_featurizer import \
CountVectorsFeaturizer
ftr = CountVectorsFeaturizer({"token_pattern": r'(?u)\b\w+\b'})
# using empty string instead of real text string to make sure
# count vector only can come from `tokens` feature.
# using `message.text` can not get correct result
message = Message("")
tokens_feature = [Token(i, 0) for i in tokens]
message.set("tokens", tokens_feature)
message.set("intent", "bla") # this is needed for a valid training example
data = TrainingData([message])
ftr.train(data)
ftr.process(message)
assert np.all(message.get("text_features")[0] == expected)
def load_rasa_data(filename):
# type: (Text) -> TrainingData
"""Loads training data stored in the rasa NLU data format."""
with io.open(filename, encoding="utf-8-sig") as f:
data = json.loads(f.read())
validate_rasa_nlu_data(data)
common = data['rasa_nlu_data'].get("common_examples", list())
intent = data['rasa_nlu_data'].get("intent_examples", list())
entity = data['rasa_nlu_data'].get("entity_examples", list())
if intent or entity:
logger.warn(
"DEPRECATION warning: Data file contains 'intent_examples' or 'entity_examples' which will be "
+
"removed in the future. Consider putting all your examples into the 'common_examples' section."
)
all_examples = common + intent + entity
return TrainingData(all_examples)
def read(self, fn: Text, **kwargs: Any) -> 'TrainingData':
"""Loads training data stored in the Dialogflow data format."""
from rasa_nlu.training_data import TrainingData
language = kwargs["language"]
fformat = kwargs["fformat"]
if fformat not in {DIALOGFLOW_INTENT, DIALOGFLOW_ENTITIES}:
raise ValueError("fformat must be either {}, or {}"
"".format(DIALOGFLOW_INTENT, DIALOGFLOW_ENTITIES))
root_js = utils.read_json_file(fn)
examples_js = self._read_examples_js(fn, language, fformat)
if not examples_js:
logger.warning("No training examples found for dialogflow file {}!"
"".format(fn))
return TrainingData()
elif fformat == DIALOGFLOW_INTENT:
return self._read_intent(root_js, examples_js)
elif fformat == DIALOGFLOW_ENTITIES:
return self._read_entities(root_js, examples_js)