def load_api_data(self, files):
for filename in files:
with codecs.open(filename, encoding="utf-8-sig") as f:
data = json.loads(f.read())
# get only intents, skip the rest. The property name is the target class
if "userSays" in data:
intent = data.get("name")
for s in data["userSays"]:
text = "".join(map(lambda chunk: chunk["text"], s.get("data")))
# add entities to each token, if available
entities = []
for e in filter(lambda chunk: "alias" in chunk or "meta" in chunk, s.get("data")):
start = text.find(e["text"])
end = start + len(e["text"])
val = text[start:end]
entities.append(
{
"entity": e["alias"] if "alias" in e else e["meta"],
"value": val,
"start": start,
"end": end
}
)
self.intent_examples.append({"text": text, "intent": intent})
self.entity_examples.append({"text": text, "intent": intent, "entities": entities})
# create synonyms dictionary
if "name" in data and "entries" in data:
for entry in data["entries"]:
if "value" in entry and "synonyms" in entry:
for synonym in entry["synonyms"]:
util.add_entities_if_synonyms(self.entity_synonyms, synonym, entry["value"])
def load_data(self, filename):
data = json.loads(open(filename, 'rb').read())
common = data['rasa_nlu_data'].get("common_examples", list())
intent = data['rasa_nlu_data'].get("intent_examples", list())
entity = data['rasa_nlu_data'].get("entity_examples", list())
self.intent_examples = intent + common
self.entity_examples = entity + common
for example in self.entity_examples:
for entity in example["entities"]:
entity_val = example["text"][entity["start"]:entity["end"]]
util.add_entities_if_synonyms(self.entity_synonyms, entity_val,
entity.get("value"))
def load_wit_data(self, filename):
with codecs.open(filename, encoding="utf-8-sig") as f:
data = json.loads(f.read())
for s in data["data"]:
entities = s.get("entities")
if entities is None:
continue
text = s.get("text")
intents = [e["value"] for e in entities if e["entity"] == 'intent']
intent = intents[0] if intents else 'None'
entities = [e for e in entities if ("start" in e and "end" in e)]
for e in entities:
e["value"] = e["value"][1:-1]
# create synonyms dictionary
text_value = text[e["start"]:e["end"]]
util.add_entities_if_synonyms(self.entity_synonyms, text_value, e["value"])
self.intent_examples.append({"text": text, "intent": intent})
self.entity_examples.append({"text": text, "intent": intent, "entities": entities})
