def test_preprocess(self):
# Given
language = LANGUAGE_EN
resources = {
STEMS: {
"beautiful": "beauty",
"birdy": "bird",
"entity": "ent"
},
WORD_CLUSTERS: {
"my_word_clusters": {
"beautiful": "cluster_1",
"birdy": "cluster_2",
"entity": "cluster_3"
}
},
STOP_WORDS: set()
}
dataset_stream = io.StringIO("""
---
type: intent
name: intent1
utterances:
- dummy utterance
---
type: entity
name: entity_1
values:
- [entity 1, alternative entity 1]
- [éntity 1, alternative entity 1]
---
type: entity
name: entity_2
values:
- entity 1
- [Éntity 2, Éntity_2, Alternative entity 2]""")
dataset = Dataset.from_yaml_files("en", [dataset_stream]).json
custom_entity_parser = CustomEntityParser.build(
dataset, CustomEntityParserUsage.WITH_STEMS, resources)
builtin_entity_parser = BuiltinEntityParser.build(dataset, language)
utterances = [
text_to_utterance("hÉllo wOrld Éntity_2"),
text_to_utterance("beauTiful World entity 1"),
text_to_utterance("Bird bïrdy"),
text_to_utterance("Bird birdy"),
]
config = TfidfVectorizerConfig(use_stemming=True,
word_clusters_name="my_word_clusters")
vectorizer = TfidfVectorizer(
config=config,
custom_entity_parser=custom_entity_parser,
builtin_entity_parser=builtin_entity_parser,
resources=resources)
vectorizer._language = language
vectorizer.builtin_entity_scope = {"snips/number"}
# When
processed_data = vectorizer._preprocess(utterances)
processed_data = list(zip(*processed_data))
# Then
u_0 = {"data": [{"text": "hello world entity_2"}]}
u_1 = {"data": [{"text": "beauty world ent 1"}]}
u_2 = {"data": [{"text": "bird bird"}]}
u_3 = {"data": [{"text": "bird bird"}]}
ent_0 = {
"entity_kind": "entity_2",
"value": "entity_2",
"resolved_value": "Éntity 2",
"range": {
"start": 12,
"end": 20
}
}
num_0 = {
"entity_kind": "snips/number",
"value": "2",
"resolved_value": {
"value": 2.0,
"kind": "Number"
},
"range": {
"start": 19,
"end": 20
}
}
ent_11 = {
"entity_kind": "entity_1",
"value": "ent 1",
"resolved_value": "entity 1",
"range": {
"start": 13,
"end": 18
}
}
ent_12 = {
"entity_kind": "entity_2",
"value": "ent 1",
"resolved_value": "entity 1",
"range": {
"start": 13,
"end": 18
}
}
num_1 = {
"entity_kind": "snips/number",
"value": "1",
"range": {
"start": 23,
"end": 24
},
"resolved_value": {
"value": 1.0,
"kind": "Number"
},
}
expected_data = [(u_0, [num_0], [ent_0], []),
(u_1, [num_1], [ent_11,
ent_12], ["cluster_1", "cluster_3"]),
(u_2, [], [], []), (u_3, [], [], ["cluster_2"])]
self.assertSequenceEqual(expected_data, processed_data)
def test_preprocess_for_training(self):
# Given
language = LANGUAGE_EN
resources = {
STEMS: {
"beautiful": "beauty",
"birdy": "bird",
"entity": "ent"
},
WORD_CLUSTERS: {
"my_word_clusters": {
"beautiful": "cluster_1",
"birdy": "cluster_2",
"entity": "cluster_3"
}
},
STOP_WORDS: set()
}
dataset_stream = io.StringIO("""
---
type: intent
name: intent1
utterances:
- dummy utterance
---
type: entity
name: entity_1
automatically_extensible: false
use_synononyms: false
matching_strictness: 1.0
values:
- [entity 1, alternative entity 1]
- [éntity 1, alternative entity 1]
---
type: entity
name: entity_2
automatically_extensible: false
use_synononyms: true
matching_strictness: 1.0
values:
- entity 1
- [Éntity 2, Éntity_2, Alternative entity 2]""")
dataset = Dataset.from_yaml_files("en", [dataset_stream]).json
custom_entity_parser = CustomEntityParser.build(
dataset, CustomEntityParserUsage.WITH_STEMS, resources)
builtin_entity_parser = BuiltinEntityParser.build(dataset, language)
utterances = [{
"data": [{
"text": "hÉllo wOrld "
}, {
"text": " yo "
}, {
"text": " yo "
}, {
"text": "yo "
}, {
"text": "Éntity_2",
"entity": "entity_2"
}, {
"text": " "
}, {
"text": "Éntity_2",
"entity": "entity_2"
}]
}, {
"data": [{
"text": "beauTiful World "
}, {
"text": "entity 1",
"entity": "entity_1"
}, {
"text": " "
}, {
"text": "2",
"entity": "snips/number"
}]
}, {
"data": [{
"text": "Bird bïrdy"
}]
}, {
"data": [{
"text": "Bird birdy"
}]
}]
config = TfidfVectorizerConfig(use_stemming=True,
word_clusters_name="my_word_clusters")
vectorizer = TfidfVectorizer(
config=config,
custom_entity_parser=custom_entity_parser,
builtin_entity_parser=builtin_entity_parser,
resources=resources)
vectorizer._language = language
# When
processed_data = vectorizer._preprocess(utterances, training=True)
processed_data = list(zip(*processed_data))
# Then
u_0 = {
"data": [{
"text": "hello world"
}, {
"text": "yo"
}, {
"text": "yo"
}, {
"text": "yo"
}, {
"text": "entity_2",
"entity": "entity_2"
}, {
"text": ""
}, {
"text": "entity_2",
"entity": "entity_2"
}]
}
u_1 = {
"data": [{
"text": "beauty world"
}, {
"text": "ent 1",
"entity": "entity_1"
}, {
"text": ""
}, {
"text": "2",
"entity": "snips/number"
}]
}
u_2 = {"data": [{"text": "bird bird"}]}
ent_00 = {
"entity_kind": "entity_2",
"value": "Éntity_2",
"range": {
"start": 23,
"end": 31
}
}
ent_01 = {
"entity_kind": "entity_2",
"value": "Éntity_2",
"range": {
"start": 32,
"end": 40
}
}
ent_1 = {
"entity_kind": "entity_1",
"value": "entity 1",
"range": {
"start": 16,
"end": 24
}
}
num_1 = {
"entity_kind": "snips/number",
"value": "2",
"range": {
"start": 25,
"end": 26
}
}
expected_data = [(u_0, [], [ent_00, ent_01], []),
(u_1, [num_1], [ent_1], ["cluster_1", "cluster_3"]),
(u_2, [], [], []), (u_2, [], [], ["cluster_2"])]
self.assertSequenceEqual(expected_data, processed_data)