def test_crf_extractor(spacy_nlp):
from rasa_nlu.extractors.crf_entity_extractor import CRFEntityExtractor
ext = CRFEntityExtractor()
examples = [
Message("anywhere in the west", {
"intent": "restaurant_search",
"entities": [{"start": 16, "end": 20, "value": "west", "entity": "location"}],
"spacy_doc": spacy_nlp("anywhere in the west")
}),
Message("central indian restaurant", {
"intent": "restaurant_search",
"entities": [{"start": 0, "end": 7, "value": "central", "entity": "location"}],
"spacy_doc": spacy_nlp("central indian restaurant")
})]
config = {"ner_crf": {"BILOU_flag": True, "features": ext.crf_features}}
ext.train(TrainingData(training_examples=examples), config)
sentence = 'anywhere in the west'
crf_format = ext._from_text_to_crf(Message(sentence, {"spacy_doc": spacy_nlp(sentence)}))
assert [word[0] for word in crf_format] == ['anywhere', 'in', 'the', 'west']
feats = ext._sentence_to_features(crf_format)
assert 'BOS' in feats[0]
assert 'EOS' in feats[-1]
assert feats[1]['0:low'] == "in"
sentence = 'anywhere in the west'
ext.extract_entities(Message(sentence, {"spacy_doc": spacy_nlp(sentence)}))
def test_unintentional_synonyms_capitalized(component_builder):
_config = utilities.base_test_conf("all_components")
ner_syn = component_builder.create_component("ner_synonyms", _config)
examples = [
Message(
"Any Mexican restaurant will do", {
"intent":
"restaurant_search",
"entities": [{
"start": 4,
"end": 11,
"value": "Mexican",
"entity": "cuisine"
}]
}),
Message(
"I want Tacos!", {
"intent":
"restaurant_search",
"entities": [{
"start": 7,
"end": 12,
"value": "Mexican",
"entity": "cuisine"
}]
})
]
ner_syn.train(TrainingData(training_examples=examples), _config)
assert ner_syn.synonyms.get("mexican") is None
assert ner_syn.synonyms.get("tacos") == "Mexican"
def test_count_vector_featurizer_using_tokens(tokens, expected):
from rasa_nlu.featurizers.count_vectors_featurizer import \
CountVectorsFeaturizer
ftr = CountVectorsFeaturizer({"token_pattern": r'(?u)\b\w+\b'})
# using empty string instead of real text string to make sure
# count vector only can come from `tokens` feature.
# using `message.text` can not get correct result
tokens_feature = [Token(i, 0) for i in tokens]
train_message = Message("")
train_message.set("tokens", tokens_feature)
# this is needed for a valid training example
train_message.set("intent", "bla")
data = TrainingData([train_message])
ftr.train(data)
test_message = Message("")
test_message.set("tokens", tokens_feature)
ftr.process(test_message)
assert np.all(test_message.get("text_features") == expected)
def test_count_vector_featurizer(sentence, expected):
from rasa_nlu.featurizers.count_vectors_featurizer import \
CountVectorsFeaturizer
ftr = CountVectorsFeaturizer({"token_pattern": r'(?u)\b\w+\b'})
train_message = Message(sentence)
train_message.set("intent", "bla")
data = TrainingData([train_message])
ftr.train(data)
test_message = Message(sentence)
ftr.process(test_message)
assert np.all(test_message.get("text_features") == expected)
def test_spacy_ner_extractor(component_builder, spacy_nlp):
_config = RasaNLUModelConfig(
{"pipeline": [{
"name": "SpacyEntityExtractor"
}]})
ext = component_builder.create_component(_config.for_component(0), _config)
example = Message(
"anywhere in the West", {
"intent": "restaurant_search",
"entities": [],
"spacy_doc": spacy_nlp("anywhere in the west")
})
ext.process(example, spacy_nlp=spacy_nlp)
assert len(example.get("entities", [])) == 1
assert example.get("entities")[0] == {
'start': 16,
'extractor': 'SpacyEntityExtractor',
'end': 20,
'value': 'West',
'entity': 'LOC',
'confidence': None
}
# Test dimension filtering includes only specified dimensions
example = Message(
"anywhere in the West with Sebastian Thrun", {
"intent": "example_intent",
"entities": [],
"spacy_doc": spacy_nlp("anywhere in the West with Sebastian Thrun")
})
_config = RasaNLUModelConfig(
{"pipeline": [{
"name": "SpacyEntityExtractor"
}]})
_config.set_component_attr(0, dimensions=["PERSON"])
ext = component_builder.create_component(_config.for_component(0), _config)
ext.process(example, spacy_nlp=spacy_nlp)
assert len(example.get("entities", [])) == 1
assert example.get("entities")[0] == {
'start': 26,
'extractor': 'SpacyEntityExtractor',
'end': 41,
'value': 'Sebastian Thrun',
'entity': 'PERSON',
'confidence': None
}
def test_duckling_entity_extractor(component_builder):
_config = utilities.base_test_conf("all_components")
_config["duckling_dimensions"] = ["time"]
duckling = component_builder.create_component("ner_duckling", _config)
message = Message("Today is the 5th of May. Let us meet tomorrow.")
duckling.process(message)
entities = message.get("entities")
assert len(entities) == 3
# Test duckling with a defined date
message = Message("Let us meet tomorrow.", time="1381536182000") # 1381536182000 == 2013/10/12 02:03:02
duckling.process(message)
entities = message.get("entities")
assert len(entities) == 1
assert entities[0]["text"] == "tomorrow"
assert entities[0]["value"] == "2013-10-13T00:00:00.000Z"
def load_data(filename):
# type: (Text) -> TrainingData
"""Loads training data stored in the rasa NLU data format."""
with io.open(filename, encoding="utf-8-sig") as f:
data = json.loads(f.read())
common = data['rasa_nlu_data'].get("common_examples", list())
intent = data['rasa_nlu_data'].get("intent_examples", list())
entity = data['rasa_nlu_data'].get("entity_examples", list())
regex_features = data['rasa_nlu_data'].get("regex_features", list())
synonyms = data['rasa_nlu_data'].get("entity_synonyms", list())
entity_synonyms = get_entity_synonyms_dict(synonyms)
all_examples = common + intent + entity
training_examples = []
for e in all_examples:
data = e.copy()
if "text" in data:
del data["text"]
training_examples.append(Message(e["text"], data))
return TrainingData(training_examples, entity_synonyms, regex_features)
def filter_trainable_entities(
self, entity_examples: List[Message]) -> List[Message]:
"""Filters out untrainable entity annotations.
Creates a copy of entity_examples in which entities that have
`extractor` set to something other than
self.name (e.g. 'CRFEntityExtractor') are removed.
"""
filtered = []
for message in entity_examples:
entities = []
for ent in message.get("entities", []):
extractor = ent.get("extractor")
if not extractor or extractor == self.name:
entities.append(ent)
data = message.data.copy()
data['entities'] = entities
filtered.append(
Message(text=message.text,
data=data,
output_properties=message.output_properties,
time=message.time))
return filtered
def test_count_vector_featurizer_oov_token(sentence, expected):
from rasa_nlu.featurizers.count_vectors_featurizer import \
CountVectorsFeaturizer
ftr = CountVectorsFeaturizer({"token_pattern": r'(?u)\b\w+\b',
"OOV_token": '__oov__'})
train_message = Message(sentence)
# this is needed for a valid training example
train_message.set("intent", "bla")
data = TrainingData([train_message])
ftr.train(data)
test_message = Message(sentence)
ftr.process(test_message)
assert np.all(test_message.get("text_features") == expected)
def load_train_data(data):
validate_rasa_nlu_data(data)
common = data['rasa_nlu_data'].get("common_examples", list())
intent = data['rasa_nlu_data'].get("intent_examples", list())
entity = data['rasa_nlu_data'].get("entity_examples", list())
regex_features = data['rasa_nlu_data'].get("regex_features", list())
synonyms = data['rasa_nlu_data'].get("entity_synonyms", list())
entity_synonyms = get_entity_synonyms_dict(synonyms)
if intent or entity:
logger.warn(
"DEPRECATION warning: Data file contains 'intent_examples' or 'entity_examples' which will be "
+
"removed in the future. Consider putting all your examples into the 'common_examples' section."
)
all_examples = common + intent + entity
training_examples = []
for e in all_examples:
data = {}
if e.get("intent"):
data["intent"] = e["intent"]
if e.get("entities") is not None:
data["entities"] = e["entities"]
training_examples.append(Message(e["text"], data))
return TrainingData(training_examples, entity_synonyms, regex_features)
def load_rasa_data(filename):
# type: (Text) -> TrainingData
"""Loads training data stored in the rasa NLU data format."""
data = _read_json_from_file(filename)
validate_rasa_nlu_data(data)
common = data['rasa_nlu_data'].get("common_examples", list())
intent = data['rasa_nlu_data'].get("intent_examples", list())
entity = data['rasa_nlu_data'].get("entity_examples", list())
regex_features = data['rasa_nlu_data'].get("regex_features", list())
synonyms = data['rasa_nlu_data'].get("entity_synonyms", list())
entity_synonyms = get_entity_synonyms_dict(synonyms)
if intent or entity:
logger.warn("DEPRECATION warning: Data file contains 'intent_examples' "
"or 'entity_examples' which will be "
"removed in the future. Consider putting all your examples "
"into the 'common_examples' section.")
all_examples = common + intent + entity
training_examples = []
for e in all_examples:
data = e.copy()
if "text" in data:
del data["text"]
training_examples.append(Message(e["text"], data))
return TrainingData(training_examples, entity_synonyms, regex_features)
def parse(self, text, time=None):
# type: (Text) -> Dict[Text, Any]
"""Parse the input text, classify it and return pipeline result.
The pipeline result usually contains intent and entities."""
if not text:
# Not all components are able to handle empty strings. So we need
# to prevent that... This default return will not contain all
# output attributes of all components, but in the end, no one should
# pass an empty string in the first place.
output = self.default_output_attributes()
output["text"] = ""
return output
message = Message(text, self.default_output_attributes(), time=time)
print('[DEBUG] Message Output Attributes: %s' % self.default_output_attributes())
print('[DEBUG] Message Text: %s' % message.text)
print('[DEBUG] Message Data: %s' % message.data)
for component in self.pipeline:
component.process(message, **self.context)
output = self.default_output_attributes()
output.update(message.as_dict(only_output_properties=True))
return output
def _parse_intent_example(self, example_in_md):
entities = []
utter = example_in_md
match = re.search(ent_regex, utter)
while match is not None:
entity_synonym = match.groupdict()['synonym']
entity_entity = match.groupdict()['entity']
entity_value = match.groupdict()['value']
if match.groupdict()['value'] is None:
entity_value = entity_synonym
start_index = match.start()
end_index = start_index + len(entity_synonym)
entities.append({
'entity': entity_entity,
'value': entity_value,
'start': start_index,
'end': end_index
})
utter = utter[:match.start()] + entity_synonym + utter[match.end(
):]
match = re.search(ent_regex, utter)
message = Message(utter, {'intent': self.current_intent})
if len(entities) > 0:
message.set('entities', entities)
return message
def test_lookup_tables(sentence, expected, labeled_tokens, spacy_nlp):
from rasa_nlu.featurizers.regex_featurizer import RegexFeaturizer
lookups = [{
"name":
'drinks',
"elements":
["mojito", "lemonade", "sweet berry wine", "tea", "club?mate"]
}, {
"name": 'plates',
"elements": "data/test/lookup_tables/plates.txt"
}]
ftr = RegexFeaturizer(lookup_tables=lookups)
# adds tokens to the message
tokenizer = SpacyTokenizer()
message = Message(sentence)
message.set("spacy_doc", spacy_nlp(sentence))
tokenizer.process(message)
result = ftr.features_for_patterns(message)
assert np.allclose(result, expected, atol=1e-10)
# the tokenizer should have added tokens
assert len(message.get("tokens", [])) > 0
# the number of regex matches on each token should match
for i, token in enumerate(message.get("tokens")):
token_matches = token.get("pattern").values()
num_matches = sum(token_matches)
assert (num_matches == labeled_tokens.count(i))
def load_wit_data(filename):
# type: (Text) -> TrainingData
"""Loads training data stored in the WIT.ai data format."""
training_examples = []
data = _read_json_from_file(filename)
for s in data["data"]:
entities = s.get("entities")
if entities is None:
continue
text = s.get("text")
intents = [e["value"] for e in entities if e["entity"] == 'intent']
intent = intents[0].strip("\"") if intents else None
entities = [e
for e in entities
if ("start" in e and "end" in e and
e["entity"] != 'intent')]
for e in entities:
# for some reason wit adds additional quotes around entity values
e["value"] = e["value"].strip("\"")
data = {}
if intent:
data["intent"] = intent
if entities is not None:
data["entities"] = entities
training_examples.append(Message(text, data))
return TrainingData(training_examples)
def parse(self,
text: Text,
time: Optional[datetime.datetime] = None,
only_output_properties: bool = True) -> Dict[Text, Any]:
"""Parse the input text, classify it and return pipeline result.
The pipeline result usually contains intent and entities."""
if not text:
# Not all components are able to handle empty strings. So we need
# to prevent that... This default return will not contain all
# output attributes of all components, but in the end, no one
# should pass an empty string in the first place.
output = self.default_output_attributes()
output["text"] = ""
return output
message = Message(text, self.default_output_attributes(), time=time)
for component in self.pipeline:
component.process(message, **self.context)
output = self.default_output_attributes()
output.update(
message.as_dict(only_output_properties=only_output_properties))
return output
def test_regex_featurizer(sentence, expected, labeled_tokens, spacy_nlp):
from rasa_nlu.featurizers.regex_featurizer import RegexFeaturizer
patterns = [{
"pattern": '[0-9]+',
"name": "number",
"usage": "intent"
}, {
"pattern": '\\bhey*',
"name": "hello",
"usage": "intent"
}, {
"pattern": '[0-1]+',
"name": "binary",
"usage": "intent"
}]
ftr = RegexFeaturizer(known_patterns=patterns)
# adds tokens to the message
tokenizer = SpacyTokenizer()
message = Message(sentence)
message.set("spacy_doc", spacy_nlp(sentence))
tokenizer.process(message)
result = ftr.features_for_patterns(message)
assert np.allclose(result, expected, atol=1e-10)
# the tokenizer should have added tokens
assert len(message.get("tokens", [])) > 0
# the number of regex matches on each token should match
for i, token in enumerate(message.get("tokens")):
token_matches = token.get("pattern").values()
num_matches = sum(token_matches)
assert (num_matches == labeled_tokens.count(i))
def test_crf_json_from_BILOU(spacy_nlp):
from rasa_nlu.extractors.crf_entity_extractor import CRFEntityExtractor
ext = CRFEntityExtractor()
ext.BILOU_flag = True
sentence = u"I need a home cleaning close-by"
doc = {"spacy_doc": spacy_nlp(sentence)}
r = ext._from_crf_to_json(Message(sentence, doc),
[{'O': 1.0},
{'O': 1.0},
{'O': 1.0},
{'B-what': 1.0},
{'L-what': 1.0},
{'B-where': 1.0},
{'I-where': 1.0},
{'L-where': 1.0}])
assert len(r) == 2, "There should be two entities"
assert r[0]["confidence"] # confidence should exist
del r[0]["confidence"]
assert r[0] == {'start': 9, 'end': 22,
'value': 'home cleaning', 'entity': 'what'}
assert r[1]["confidence"] # confidence should exist
del r[1]["confidence"]
assert r[1] == {'start': 23, 'end': 31,
'value': 'close-by', 'entity': 'where'}
def test_crf_json_from_non_BILOU(spacy_nlp):
from rasa_nlu.extractors.crf_entity_extractor import CRFEntityExtractor
ext = CRFEntityExtractor()
ext.BILOU_flag = False
sentence = u"I need a home cleaning close-by"
doc = {"spacy_doc": spacy_nlp(sentence)}
rs = ext._from_crf_to_json(Message(sentence, doc),
[{'O': 1.0},
{'O': 1.0},
{'O': 1.0},
{'what': 1.0},
{'what': 1.0},
{'where': 1.0},
{'where': 1.0},
{'where': 1.0}])
# non BILOU will split multi-word entities - hence 5
assert len(rs) == 5, "There should be five entities"
for r in rs:
assert r['confidence'] # confidence should exist
del r['confidence']
assert rs[0] == {'start': 9, 'end': 13,
'value': 'home', 'entity': 'what'}
assert rs[1] == {'start': 14, 'end': 22,
'value': 'cleaning', 'entity': 'what'}
assert rs[2] == {'start': 23, 'end': 28,
'value': 'close', 'entity': 'where'}
assert rs[3] == {'start': 28, 'end': 29,
'value': '-', 'entity': 'where'}
assert rs[4] == {'start': 29, 'end': 31,
'value': 'by', 'entity': 'where'}
def test_regex_featurizer(sentence, expected, labeled_tokens, spacy_nlp):
from rasa_nlu.featurizers.regex_featurizer import RegexFeaturizer
patterns = [
{"pattern": '[0-9]+', "name": "number", "usage": "intent"},
{"pattern": '\\bhey*', "name": "hello", "usage": "intent"}
]
ftr = RegexFeaturizer(known_patterns=patterns)
# adds tokens to the message
tokenizer = SpacyTokenizer()
message = Message(sentence)
message.set("spacy_doc", spacy_nlp(sentence))
tokenizer.process(message)
result = ftr.features_for_patterns(message)
assert np.allclose(result, expected, atol=1e-10)
# the tokenizer should have added tokens
assert len(message.get("tokens", [])) > 0
for i, token in enumerate(message.get("tokens")):
if i in labeled_tokens:
assert token.get("pattern") in [0, 1]
else:
# if the token is not part of a regex the pattern should not be set
assert token.get("pattern") is None
def _parse_intent_example(self, example_in_md):
entities = []
utter = example_in_md
for regex in [ent_regex, ent_regex_with_value]:
utter = re.sub(regex, r"\1", utter) # [text](entity) -> text
ent_matches = re.finditer(regex, example_in_md)
for matchNum, match in enumerate(ent_matches):
if 'synonym' in match.groupdict():
entity_value_in_utter = match.groupdict()['synonym']
else:
entity_value_in_utter = match.groupdict()['value']
start_index = utter.index(entity_value_in_utter)
end_index = start_index + len(entity_value_in_utter)
entities.append({
'entity': match.groupdict()['entity'],
'value': match.groupdict()['value'],
'start': start_index,
'end': end_index
})
message = Message(utter, {'intent': self.current_intent})
if len(entities) > 0:
message.set('entities', entities)
return message
def read_from_json(self, js: Dict[Text, Any], **kwargs: Any):
"""Loads training data stored in the WIT.ai data format."""
from rasa_nlu.training_data import Message, TrainingData
training_examples = []
for s in js["data"]:
entities = s.get("entities")
if entities is None:
continue
text = s.get("text")
intents = [e["value"] for e in entities if e["entity"] == 'intent']
intent = intents[0].strip("\"") if intents else None
entities = [
e for e in entities
if ("start" in e and "end" in e and e["entity"] != 'intent')
]
for e in entities:
# for some reason wit adds additional quotes around entities
e["value"] = e["value"].strip("\"")
data = {}
if intent:
data["intent"] = intent
if entities is not None:
data["entities"] = entities
training_examples.append(Message(text, data))
return TrainingData(training_examples)
def load_rasa_data(filename):
# type: (Text) -> TrainingData
"""Loads training data stored in the rasa NLU data format."""
with io.open(filename, encoding="utf-8-sig") as f:
data = json.loads(f.read())
validate_rasa_nlu_data(data)
common = data['rasa_nlu_data'].get("common_examples", list())
intent = data['rasa_nlu_data'].get("intent_examples", list())
entity = data['rasa_nlu_data'].get("entity_examples", list())
regex_features = data['rasa_nlu_data'].get("regex_features", list())
synonyms = data['rasa_nlu_data'].get("entity_synonyms", list())
entity_synonyms = get_entity_synonyms_dict(synonyms)
if intent or entity:
logger.warn(
"DEPRECATION warning: Data file contains 'intent_examples' or 'entity_examples' which will be "
+
"removed in the future. Consider putting all your examples into the 'common_examples' section."
)
all_examples = common + intent + entity
training_examples = []
for e in all_examples:
data = {}
if e.get("intent"):
data["intent"] = e["intent"]
if e.get("entities") is not None:
data["entities"] = e["entities"]
training_examples.append(Message(e["text"], data))
return TrainingData(training_examples, entity_synonyms, regex_features)
def test_duckling_entity_extractor(component_builder):
_config = utilities.base_test_conf("all_components")
_config["duckling_dimensions"] = ["time"]
duckling = component_builder.create_component("ner_duckling", _config)
message = Message("Today is the 5th of May. Let us meet tomorrow.")
duckling.process(message)
entities = message.get("entities")
assert len(entities) == 3
def test_count_vector_featurizer(sentence, expected):
from rasa_nlu.featurizers.count_vectors_featurizer import \
CountVectorsFeaturizer
ftr = CountVectorsFeaturizer({"min_ngram": 1,
"max_ngram": 2,
"analyzer": 'char'})
train_message = Message(sentence)
# this is needed for a valid training example
train_message.set("intent", "bla")
data = TrainingData([train_message])
ftr.train(data)
test_message = Message(sentence)
ftr.process(test_message)
assert np.all(test_message.get("text_features") == expected)
def test_crf_extractor(spacy_nlp, ner_crf_pos_feature_config):
from rasa_nlu.extractors.crf_entity_extractor import CRFEntityExtractor
ext = CRFEntityExtractor(component_config=ner_crf_pos_feature_config)
examples = [
Message("anywhere in the west", {
"intent": "restaurant_search",
"entities": [{"start": 16, "end": 20,
"value": "west", "entity": "location"}],
"spacy_doc": spacy_nlp("anywhere in the west")
}),
Message("central indian restaurant", {
"intent": "restaurant_search",
"entities": [
{"start": 0, "end": 7, "value": "central",
"entity": "location", "extractor": "random_extractor"},
{"start": 8, "end": 14, "value": "indian",
"entity": "cuisine", "extractor": "ner_crf"}
],
"spacy_doc": spacy_nlp("central indian restaurant")
})]
# uses BILOU and the default features
ext.train(TrainingData(training_examples=examples), RasaNLUModelConfig())
sentence = 'anywhere in the west'
doc = {"spacy_doc": spacy_nlp(sentence)}
crf_format = ext._from_text_to_crf(Message(sentence, doc))
assert [word[0] for word in crf_format] == ['anywhere', 'in', 'the', 'west']
feats = ext._sentence_to_features(crf_format)
assert 'BOS' in feats[0]
assert 'EOS' in feats[-1]
assert feats[1]['0:low'] == "in"
sentence = 'anywhere in the west'
ext.extract_entities(Message(sentence, {"spacy_doc": spacy_nlp(sentence)}))
filtered = ext.filter_trainable_entities(examples)
assert filtered[0].get('entities') == [
{"start": 16, "end": 20, "value": "west", "entity": "location"}
], 'Entity without extractor remains'
assert filtered[1].get('entities') == [
{"start": 8, "end": 14,
"value": "indian", "entity": "cuisine", "extractor": "ner_crf"}
], 'Only ner_crf entity annotation remains'
assert examples[1].get('entities')[0] == {
"start": 0, "end": 7,
"value": "central", "entity": "location",
"extractor": "random_extractor"
}, 'Original examples are not mutated'
def test_ngram_featurizer(spacy_nlp):
from rasa_nlu.featurizers.ngram_featurizer import NGramFeaturizer
ftr = NGramFeaturizer({"max_number_of_ngrams": 10})
# ensures that during random sampling of the ngram CV we don't end up
# with a one-class-split
repetition_factor = 5
greet = {"intent": "greet", "text_features": [0.5]}
goodbye = {"intent": "goodbye", "text_features": [0.5]}
labeled_sentences = [
Message("heyheyheyhey", greet),
Message("howdyheyhowdy", greet),
Message("heyhey howdyheyhowdy", greet),
Message("howdyheyhowdy heyhey", greet),
Message("astalavistasista", goodbye),
Message("astalavistasista sistala", goodbye),
Message("sistala astalavistasista", goodbye),
] * repetition_factor
for m in labeled_sentences:
m.set("spacy_doc", spacy_nlp(m.text))
ftr.min_intent_examples_for_ngram_classification = 2
ftr.train_on_sentences(labeled_sentences)
assert len(ftr.all_ngrams) > 0
assert ftr.best_num_ngrams > 0
def test_duckling_entity_extractor(component_builder):
_config = RasaNLUModelConfig({"pipeline": [{"name": "ner_duckling"}]})
_config.set_component_attr("ner_duckling", dimensions=["time"])
duckling = component_builder.create_component("ner_duckling", _config)
message = Message("Today is the 5th of May. Let us meet tomorrow.")
duckling.process(message)
entities = message.get("entities")
assert len(entities) == 3
# Test duckling with a defined date
# 1381536182000 == 2013/10/12 02:03:02
message = Message("Let us meet tomorrow.", time="1381536182000")
duckling.process(message)
entities = message.get("entities")
assert len(entities) == 1
assert entities[0]["text"] == "tomorrow"
assert entities[0]["value"] == "2013-10-13T00:00:00.000Z"
def test_duckling_entity_extractor_and_synonyms(component_builder):
_config = utilities.base_test_conf("all_components")
_config["duckling_dimensions"] = ["number"]
duckling = component_builder.create_component("ner_duckling", _config)
synonyms = component_builder.create_component("ner_synonyms", _config)
message = Message("He was 6 feet away")
duckling.process(message)
synonyms.process(message) # checks that the synonym processor can handle entities that have int values
assert message is not None
def _parse_training_example(self, example):
"""Extract entities and synonyms, and convert to plain text."""
entities = self._find_entities_in_training_example(example)
plain_text = re.sub(ent_regex, lambda m: m.groupdict()['entity_text'], example)
self._add_synonyms(plain_text, entities)
message = Message(plain_text, {'intent': self.current_title})
if len(entities) > 0:
message.set('entities', entities)
return message
