class NLP():
def __init__(self):
# load language resources
load_resources(u"en")
# create NLU Engine
self.engine = SnipsNLUEngine(config=CONFIG_EN)
# train engine
"""
with io.open("dataset.json") as f:
dataset = json.load(f)
print('start training')
self.engine.fit(dataset=dataset)
print('finished training')
self.engine.persist('nlu_engine')
"""
self.engine = SnipsNLUEngine.from_path('nlu_engine')
print('snips engine ready')
def parse(self, txt):
parsed = self.engine.parse(txt)
#print(json.dumps(parsed['slots'], indent=2))
slots = parsed['slots']
try:
for slot in slots:
output = slot['value']['value']
return True, output
except Exception as e:
return False, e
class SnipsService(ApiService):
def __init__(self,
classes,
model_path=None,
max_api_calls=None,
verbose=False):
super().__init__(classes, max_api_calls, verbose)
load_resources('en')
if model_path:
self.load_model(model_path)
else:
self.engine = SnipsNLUEngine(config=CONFIG_EN)
def train_model(self, dataset):
self.engine.fit(dataset)
def train_model_from_file(self, dataset_path):
with io.open(dataset_path) as f:
self.train_model(json.load(f))
def save_model(self, model_path):
self.engine.persist(model_path)
def load_model(self, model_path):
self.engine = SnipsNLUEngine.from_path(model_path)
def predict(self, utterance):
result = self.engine.parse(utterance)
try:
return result['intent']['intentName']
except Exception as e:
print('ERR:', e)
print('Failed to parse: "{}"'.format(utterance))
print(result)
return None
def nluparse(text):
load_resources(sample_dataset["language"])
nlu_engine = SnipsNLUEngine(config=config)
nlu_engine.fit(sample_dataset)
# text = "Show me jobs in LA for today"
parsing = nlu_engine.parse(text)
return json.dumps(parsing, indent=2)
class ConfigEngine(Engine):
def __init__(self):
self.engine = None
self.config = config
def fit(self, dataset):
self.engine = SnipsNLUEngine(self.config).fit(dataset)
return self
def parse(self, text):
return self.engine.parse(text)
def conversacion(m):
with io.open("training.json") as f:
sample_dataset = json.load(f)
load_resources("en")
nlu_engine = SnipsNLUEngine(config=CONFIG_EN)
nlu_engine.fit(sample_dataset)
text = (u"" + m.text.lower() + "")
listaResultado = nlu_engine.parse(text)
return procesarRespuesta(listaResultado)
def make_nlu_model_json(fname):
docs = yaml.load_all(stream)
ll = []
for doc in docs:
i = {}
for k, v in doc.items():
i[k] = v
ll.append(i)
dataset = Dataset.from_yaml_files("en", [ll])
nlu_engine = SnipsNLUEngine(config=CONFIG_EN)
nlu_engine = nlu_engine.fit(dataset)
text = "Please turn the light on in the kitchen"
parsing = nlu_engine.parse(text)
def runEngine(query):
with io.open("dataset.json") as f:
dataset = json.load(f)
load_resources("en")
#with io.open("config_en.json") as f:
# config = json.load(f)
#engine = SnipsNLUEngine(config=config)
engine = SnipsNLUEngine(config=CONFIG_EN)
engine.fit(dataset)
parsing = engine.parse(query)
return json.dumps(parsing, indent=2)
def test_default_configs_should_work(self):
# Given
dataset = deepcopy(WEATHER_DATASET)
for language in get_all_languages():
# When
config = DEFAULT_CONFIGS.get(language)
self.assertIsNotNone(config,
"Missing default config for '%s'" % language)
dataset[LANGUAGE] = language
engine = SnipsNLUEngine(config).fit(dataset)
result = engine.parse("Please give me the weather in Paris")
# Then
intent_name = result[RES_INTENT][RES_INTENT_NAME]
self.assertEqual("SearchWeatherForecast", intent_name)
class NLU:
def __init__(self):
self.sample_dataset = NLU.load_dataset()
@staticmethod
def load_dataset():
"""
Load the sample dataset which will be used to train the snipsnlu NLP
engine.
:return:
"""
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
try:
# with io.open(str(Path('app', 'static', 'samples', 'sample_dataset.json'))) as fr:
# sample_dataset = json.load(fr)
# return sample_dataset
with open(f'{BASE_DIR}/static/sample/test.json',
'r',
encoding='utf-8') as fr:
sample_dataset = json.load(fr)
return sample_dataset
except Exception as e:
print("Could not load dataset {}".format(str(e)))
def train_engine(self):
"""
# Setup the snipsnlu NLP engine and pass the training data.
:return:
"""
load_resources("en")
self.nlu_engine = SnipsNLUEngine(config=CONFIG_EN)
self.nlu_engine.fit(self.sample_dataset)
def parse_sentence(self, sentence):
"""
Get the sentence and parse it to get the result.
The sentence is a query made in any natural language(for now we are
setting this language as english) and the result is the json string
with the parsed help of trained engine and the possible correct
prediction of what the query actually meant.
:param sentence:
:return:
"""
parsing = self.nlu_engine.parse(sentence)
return parsing
def test_sample_configs_should_work(self):
# Given
dataset = self.sample_dataset
for language in get_all_languages():
# When
config_file = "config_%s.json" % language
config_path = os.path.join(SAMPLES_PATH, "configs", config_file)
with io.open(config_path) as f:
config = json.load(f)
dataset[LANGUAGE] = language
engine = SnipsNLUEngine(config).fit(dataset)
result = engine.parse("Please give me the weather in Paris")
# Then
intent_name = result[RES_INTENT][RES_INTENT_NAME]
self.assertEqual("sampleGetWeather", intent_name)
def debug_training(dataset_path, config_path=None):
if config_path is None:
config = NLUEngineConfig()
else:
with io.open(config_path, "r", encoding="utf8") as f:
config = NLUEngineConfig.from_dict(json.load(f))
with io.open(os.path.abspath(dataset_path), "r", encoding="utf8") as f:
dataset = json.load(f)
engine = SnipsNLUEngine(config).fit(dataset)
while True:
query = input("Enter a query (type 'q' to quit): ").strip()
if isinstance(query, bytes):
query = query.decode("utf8")
if query == "q":
break
print(json.dumps(engine.parse(query), indent=2))
def test_keywords_slot_filler_should_work_in_engine(self):
# Given
dataset_stream = io.StringIO("""
---
type: intent
name: SetLightColor
utterances:
- set the light to [color](blue) in the [room](kitchen)
- please make the lights [color](red) in the [room](bathroom)""")
dataset = Dataset.from_yaml_files("en", [dataset_stream]).json
intent = "SetLightColor"
slot_filler_config = {
"unit_name": "keyword_slot_filler",
"lowercase": True
}
parser_config = ProbabilisticIntentParserConfig(
slot_filler_config=slot_filler_config)
engine_config = NLUEngineConfig([parser_config])
engine = SnipsNLUEngine(engine_config).fit(dataset, intent)
text = "I want Red lights in the kitchen now"
# When
res = engine.parse(text)
# Then
expected_slots = [
custom_slot(
unresolved_slot(match_range={
START: 7,
END: 10
},
value="Red",
entity="color",
slot_name="color"), "red"),
custom_slot(
unresolved_slot(match_range={
START: 25,
END: 32
},
value="kitchen",
entity="room",
slot_name="room"))
]
self.assertListEqual(expected_slots, res["slots"])
def imprimirmensaje(message):
campo = ""
predicado = ""
lista = []
chatid = message.chat.id
with io.open('dataset.json') as file:
dataset = json.load(file)
engine = SnipsNLUEngine()
engine.fit(dataset)
parsing = engine.parse(unicode(message.text))
temp = json.dumps(parsing, indent=2)
# try:
intentName = parsing["intent"]["intentName"]
entidad = ""
try:
enti = parsing["slots"][0]
entidad = enti["rawValue"]
except:
pass
if len(intentName) > 0 and len(entidad) > 0:
print("good")
if intentName == "descripcion":
print("si entraaaaaaaaaaaaaaaaaaa")
predicado = "http://usefulinc.com/ns/doap#description"
campo = consultaSparql1(entidad, predicado)
elif intentName == "imagen":
predicado = "http://schema.org/image"
campo = consultaSparql1(entidad, predicado)
elif intentName == "igualEn":
predicado = "http://www.w3.org/2002/07/owl#sameAs"
campo = consultaSparql1(entidad, predicado)
elif intentName == "igualKi":
predicado = "http://www.w3.org/2002/07/owl#sameAs"
campo = consultaSparql1(entidad, predicado)
elif intentName == "nombreCientifico":
predicado = "http://lod.taxonconcept.org/ontology/txn.owl#scientificName"
campo = consultaSparql1(entidad, predicado)
bot.send_message(chatid, campo)
else:
print("bad")
lista = consultaSparql()
bot.send_message(message, "perro")
def debug_training(dataset_path, config_path=None):
with Path(dataset_path).open("r", encoding="utf8") as f:
dataset = json.load(f)
load_resources(dataset["language"])
config = None
if config_path is not None:
with Path(config_path).open("r", encoding="utf8") as f:
config = NLUEngineConfig.from_dict(json.load(f))
engine = SnipsNLUEngine(config).fit(dataset)
while True:
query = input("Enter a query (type 'q' to quit): ").strip()
if isinstance(query, bytes):
query = query.decode("utf8")
if query == "q":
break
print(json.dumps(engine.parse(query), indent=2))
def debug_training(dataset_path, config_path=None):
with io.open(os.path.abspath(dataset_path), "r", encoding="utf8") as f:
dataset = json.load(f)
load_resources(dataset["language"])
if config_path is None:
config = NLUEngineConfig()
else:
with io.open(config_path, "r", encoding="utf8") as f:
config = NLUEngineConfig.from_dict(json.load(f))
engine = SnipsNLUEngine(config).fit(dataset)
while True:
query = input("Enter a query (type 'q' to quit): ").strip()
if isinstance(query, bytes):
query = query.decode("utf8")
if query == "q":
break
print(json.dumps(engine.parse(query), indent=2))
class LanguageParser(pluginmanager.IPlugin, PluginStorage):
"""
interface to parse input text
"""
def __init__(self):
super(pluginmanager.IPlugin, self).__init__()
self._plugins = {}
self._pre_train_json = dict()
self._pre_train_json['intents'] = {}
self._pre_train_json['entities'] = {}
self._pre_train_json['language'] = 'en'
self.nlu_engine = SnipsNLUEngine()
def train(self, plugins):
self._generate_pre_train_json(plugins)
self.nlu_engine.fit(self._pre_train_json)
def _generate_pre_train_json(self, plugins):
for plugin in plugins:
intent = dict()
intent['utterances'] = list()
_data = list()
_data.append(dict({'text': plugin.get_name()}))
intent['utterances'].append(dict({"data": _data}))
intent_name = '_'.join(re.findall(r"[\w']+", plugin.get_name()))
self._pre_train_json['intents'][intent_name] = intent
self._plugins[intent_name] = plugin
# handle sub commands (recursive)
self._generate_pre_train_json(plugin.get_plugins().values())
def identify_action(self, action):
parsed_action = self.nlu_engine.parse(action)
print(parsed_action)
intent_name = parsed_action['intent']['intentName']
if intent_name not in self._plugins:
return None
return self._plugins[intent_name]
def test_default_configs_should_work(self):
# Given
dataset_stream = io.StringIO("""
---
type: intent
name: TurnLightOn
utterances:
- turn on the lights
- please switch on the light
- switch the light on
- can you turn the light on ?
- I need you to turn on the lights
---
type: intent
name: GetWeather
utterances:
- what is the weather today
- What's the weather in tokyo today?
- Can you tell me the weather please ?
- what is the weather forecast for this weekend""")
dataset = Dataset.from_yaml_files("en", [dataset_stream]).json
for language in get_all_languages():
# When
config = DEFAULT_CONFIGS.get(language)
self.assertIsNotNone(config, "Missing default config for '%s'"
% language)
dataset[LANGUAGE] = language
shared = self.get_shared_data(dataset)
engine = SnipsNLUEngine(config, **shared).fit(dataset)
result = engine.parse("Please give me the weather in Paris")
# Then
self.assertIsNotNone(result[RES_INTENT])
intent_name = result[RES_INTENT][RES_INTENT_NAME]
self.assertEqual("GetWeather", intent_name)
class NLP():
def __init__(self):
# load language resources
load_resources(u"en")
# create NLU Engine
self.engine = SnipsNLUEngine(config=CONFIG_EN)
# train engine
"""
with io.open("snips/nlp_db.json") as f:
dataset = json.load(f)
print('start training')
self.engine.fit(dataset=dataset)
print('finished training')
self.engine.persist('nlu_engine')
"""
engine_location = rospy.get_param("~engine_location")
self.engine = SnipsNLUEngine.from_path(engine_location)
print('snips engine ready')
def parse(self, txt):
parsed = self.engine.parse(txt.decode("utf-8"))
#print(json.dumps(parsed['slots'], indent=2))
slots = parsed['slots']
pair = dict()
try:
for slot in slots:
pair[slot['slotName'].encode(
"ascii")] = slot['value']['value'].encode("ascii")
return True, pair
except Exception as e:
return False, e
class __impl:
__nlu_engine = None
def __init__(self):
print('Load NLU Engine')
print(
'-----------------------------------------------------------------'
)
try:
with io.open("oms_dataset.json") as f:
dataset = json.load(f)
except:
print('I/O error({0}): {1}')
sys.exit()
load_resources('snips_nlu_en')
self.__nlu_engine = SnipsNLUEngine(config=CONFIG_EN)
self.__nlu_engine.fit(dataset)
self.__nlu_engine.to_byte_array()
def parse_text(self, text):
return self.__nlu_engine.parse(text)
nlu_engine = SnipsNLUEngine(config=CONFIG_EN)
nlu_engine.fit(sample_dataset)
nlp = spacy.load('en_core_web_sm')
#################Loading Files START######################
f = open("/home/apttus/demo/docs/3PPAgreement/3PPresult.txt",
"r",
encoding='UTF8')
content = f.read()
data = content.split("=====================================================")
print("\n***************************")
for paragraphs in data:
paragraph = paragraphs.split("\n\n")
blob = TextBlob(paragraph[2])
parsing = nlu_engine.parse(paragraph[2])
print(paragraph[2])
print("\n\n\n")
if ('intent' in parsing):
#print ("category "+parsing["intent"]["intentName"])
print(json.dumps(parsing, indent=2))
else:
print("Null Category")
print("\n\n\n")
doc = nlp(paragraph[2])
print("Listing Entities ---------------------------------\n")
for entity in doc.ents:
print(entity.text, entity.label_)
print("\n***************************")
class BrewSnips:
def __init__(self, json_arg):
self.nlu_engine = None
self.json_arg = json_arg
def simple_text_cleaner(self, query_text):
"""
Replace, encode, remove unnecessary texts to ensure alignment w/ yaml format.
:param query_text: ( String ) Query
:return: ( String ) Cleaned-format query
"""
# Encode into "utf-8" format
query_text = query_text.encode(encoding='utf-8').decode("utf-8")
# Replace $, %, + into relative words
query_text = query_text.replace("$", "")
query_text = query_text.replace("%", "percent")
query_text = query_text.replace("+", "plus")
# Remove punctuations
query_text = query_text.translate(
str.maketrans('', '', string.punctuation))
# Replace "’" to avoid parsing error
query_text = query_text.replace("’", "")
return query_text
def parse_snips_intent(self):
"""
Parse original data.json into Snips NLU Engine Training Data in yaml format.
Convert into yaml file through command line prompt :
'snips-nlu generate-dataset en input-yaml-file > output-json-file'
"""
# Get original data.json in DataFrame
data_df = DataProcessing(
f"{getcwd()}/data_lake/{self.json_arg}").retrieve_process_json()
# Get list of Unique Intents
intent_list = list(set(data_df["Intent"]))
# Load SpaCy NLP Large Corpus
spacy_nlp_engine = load('en_core_web_lg')
# Init yaml object
yaml = ruamel.yaml.YAML()
# Set explicit start to True
yaml.explicit_start = True
# Parse by Intents
for intent_name in intent_list:
# yes and no are reserved values for yaml file.
# To avoid parsing error, "_" is added before the intent name.
if intent_name == "yes" or intent_name == "no":
intent_dict = {"type": "intent", "name": f"{intent_name}s"}
else:
intent_dict = {"type": "intent", "name": intent_name}
# Init Lists for Slots + Utterances
slots_value_list = []
utt_value_list = []
# Subset current Intent Data
subset_data = data_df[data_df["Intent"] ==
intent_name].reset_index(drop=True)
# Get current Intent Queries
intent_query_words = list(subset_data["Query"])
# Get the 4 grams and convert into a list
word_ngrams = (pd.Series(ngrams(intent_query_words, 4))).to_list()
# Random sample 80% of each Intent as training phrases for NLU Engine
sample_ngrams = sample(word_ngrams, int(len(subset_data) * 0.8))
# Start parsing each queries
for phrases in sample_ngrams:
# Join phrases back to one single sentence
full_text = " ".join(phrases)
# Parse Entity of the text through Spacy NLP Engine
parse_phrases = spacy_nlp_engine(full_text)
# Set slots
if len(parse_phrases.ents) > 0:
# Get Entity Label and Text, if any
for nlp_entity in parse_phrases.ents:
entity_label = nlp_entity.label_
entity_text = nlp_entity.text
# Construct "slot" for name and entity
slot_entities = {
"name": entity_label,
"entity": entity_label
}
# Replace text with entity label
full_text = full_text.replace(
entity_text, f"[{entity_label}]({entity_text})")
# Store "utterances" from the ngram
utt_value_list.append(full_text)
# Store unique "slots"
if slot_entities not in slots_value_list:
slots_value_list.append(slot_entities)
# Set slots in intent dictionary
if len(slots_value_list) > 0:
intent_dict["slots"] = slots_value_list
# Set utterances in intent dictionary
if len(utt_value_list) > 0:
intent_dict["utterances"] = utt_value_list
# If there's no utterances found, use the original ngrams
else:
intent_dict["utterances"] = [
" ".join(gram) for gram in sample_ngrams
]
# Append into output yaml
with open(f"{getcwd()}/data_lake/intent_ngram.yaml", "a") as file:
yaml.dump(intent_dict, file)
def get_nlu_engine(self):
"""
Get JSON file with our intents and entities tag from part of our input data.
Update the Snips NLU Engine.
:return: ( Snips NLU Object ) Customized Snips NLU Engine
"""
# Get parsed intent ngram JSON file
with io.open(f"{getcwd()}/data_lake/intent_ngram.json") as f:
custom_dataset = json.load(f)
# Init Snips NLU Engine
self.nlu_engine = SnipsNLUEngine(config=CONFIG_EN)
# Fit sample data into NLU Engine for customization
self.nlu_engine = self.nlu_engine.fit(custom_dataset)
return self.nlu_engine
def parse_intent_name_prob(self, text):
"""
Parse New IntentName and Probability based on customized Snips NLU Engine.
:param text: ( String ) Query
:return: ( List ) Intent Name and Probability Score
"""
# Parse queries to determine intent name and probability score
parsing = self.nlu_engine.parse(text)
# Get intent name
intent_name = parsing["intent"]["intentName"]
# get the probability
intent_prob = parsing["intent"]["probability"]
return [intent_name, intent_prob]
def brew_intent_score(self):
"""
Generate new intent name and probability score for each query.
Parse them into JSON format and write into a JSON file.
"""
# Get Customized Snips NLU Engine
self.get_nlu_engine()
# Get the original json data
with io.open(f"{getcwd()}/data_lake/{self.json_arg}") as f:
data_df = json.load(f)
# Convert list of lists into DataFrame w/ relative columns
data_content_df = pd.DataFrame(data_df,
columns=["Query", "Intent"])
# Set Intent Similarity Score w/ New Intent
data_content_df["NLU_Intent_Score"] = data_content_df[
"Query"].apply(self.parse_intent_name_prob)
# Split into individual columns : Intent and Score
data_content_df[["NLU_Intent", "NLU_Score"]] = pd.DataFrame(
data_content_df["NLU_Intent_Score"].tolist(),
index=data_content_df.index)
# Drop unused column
data_content_df.drop("NLU_Intent_Score", axis=1, inplace=True)
# Store all output into feather file
write_feather(data_content_df,
f"{getcwd()}/data_lake/SnipsNLUData.feather")
from __future__ import unicode_literals, print_function
import io
import json
from snips_nlu import SnipsNLUEngine, load_resources
with io.open("sample_dataset.json") as f:
sample_dataset = json.load(f)
with io.open("configs/config_en.json") as f:
config = json.load(f)
load_resources(sample_dataset["language"])
nlu_engine = SnipsNLUEngine(config=config)
nlu_engine.fit(sample_dataset)
text = "What will be the weather in San Francisco next week?"
parsing = nlu_engine.parse(text)
print(json.dumps(parsing, indent=2))
import io
import sys
import json
from snips_nlu import SnipsNLUEngine
import bs4
with io.open("train.json") as f:
sample_dataset = json.load(f)
nlu_engine = SnipsNLUEngine()
nlu_engine.fit(sample_dataset)
parsing = nlu_engine.parse(str(" ".join(sys.argv[1:])))
print(json.dumps(parsing, indent=4))
def recognize(
text: str,
engine: SnipsNLUEngine,
slots_dict: typing.Optional[typing.Dict[str, typing.List[str]]] = None,
slot_graphs: typing.Optional[typing.Dict[str, nx.DiGraph]] = None,
**parse_args,
) -> typing.List[Recognition]:
"""Recognize intent using Snips NLU."""
result = engine.parse(text, **parse_args)
intent_name = result.get("intent", {}).get("intentName")
if not intent_name:
# Recognition failure
return []
slots_dict = slots_dict or {}
slot_graphs = slot_graphs or {}
recognition = Recognition(text=text,
raw_text=text,
intent=Intent(name=intent_name, confidence=1.0))
# Replace Snips slot values with Rhasspy slot values (substituted)
for slot in result.get("slots", []):
slot_name = slot.get("slotName")
slot_value_dict = slot.get("value", {})
slot_value = slot_value_dict.get("value")
entity = Entity(
entity=slot_name,
source=slot.get("entity", ""),
value=slot_value,
raw_value=slot.get("rawValue", slot_value),
start=slot["range"]["start"],
end=slot["range"]["end"],
)
recognition.entities.append(entity)
if (not slot_name) or (not slot_value):
continue
slot_graph = slot_graphs.get(slot_name)
if not slot_graph and (slot_name in slots_dict):
# Convert slot values to graph
slot_graph = rhasspynlu.sentences_to_graph({
slot_name: [
rhasspynlu.jsgf.Sentence.parse(slot_line)
for slot_line in slots_dict[slot_name]
if slot_line.strip()
]
})
slot_graphs[slot_name] = slot_graph
entity.tokens = slot_value.split()
entity.raw_tokens = list(entity.tokens)
if slot_graph:
# Pass Snips value through graph
slot_recognitions = rhasspynlu.recognize(entity.tokens, slot_graph)
if slot_recognitions:
# Pull out substituted value and replace in Rhasspy entitiy
new_slot_value = slot_recognitions[0].text
entity.value = new_slot_value
entity.tokens = new_slot_value.split()
return [recognition]
class NLU:
__connection = Database()
__interaction = interaction()
__answers = {"NA": "Please Ask your department administrator for this"}
# def __Translate_Text (question,x='en'):
# translator = Translator()
# translated = translator. translate(question , dest=x)
# print("Source Language : " + translated.src)
# print("Destination Language : " + translated.dest)
# print("Origin Text : " + translated.origin)
# print("Translated Text : " + translated.text)
# return translated.text
def __int__(self):
self._query = ''
self.__result = ''
self._intent = ''
self._probability = 0
self._slots = {}
self.__engine = ''
self.__dataset = ''
self.__nluSlots = ''
self.language = ''
self.UserID = ''
def EngineMode(self, mode):
"""
Saving the engine to use the model for every question (Training Part)
or Use the model if it already exists (Testing Part)
:param mode: Test or Train string
:return: Fitted Engine
"""
if mode == "Train":
load_resources('snips_nlu_en')
self.__engine = SnipsNLUEngine(config=CONFIG_EN)
with io.open("dataset.json") as f:
self.__dataset = json.load(f)
self.__engine.fit(self.__dataset)
#UnComment to save the model
#self.__engine.persist("Z:\FCIS-ASU\Semester 8\ChatbotModel")
elif mode == "Test":
with io.open("dataset.json") as f:
self.__dataset = json.load(f)
self.__engine = SnipsNLUEngine.from_path(
"Z:\FCIS-ASU\Semester 8\ChatbotModel")
def setQuery(self, query, user):
# self._query , self.language = query
self._query = query
self.UserID = user
self.__excute()
self.answer()
def __excute(self):
parsing = self.__engine.parse(self._query)
self.__result = json.loads(json.dumps(parsing, indent=2))
print(self.__result)
def _getIntent(self):
try:
self._intent = self.__result['intent']['intentName']
return self._intent
except Exception as e:
return 'None'
def _getProbability(self):
self._probability = self.__result['intent']['probability']
return self._probability
#need Handling Exception
def _getSlots(self):
dic = {}
if len(self.__result['slots']) != 0:
for x in range(len(self.__result['slots'])):
if not re.search("Val", self.__result['slots'][x]['entity']):
if str(self.__result['slots'][x]['entity']).__contains__(
"snips"):
dic[self.__result['slots'][x]
['slotName']] = self.__result['slots'][x]['value'][
'value']
else:
dic[self.__result['slots'][x]
['entity']] = self.__result['slots'][x]['value'][
'value']
else:
return "No Slots"
self._slots = dic
return self._slots
def checkIntent(self):
if self._getIntent() == 'None':
self.__interaction.UnAnsweredFactory().__noIntent_insert__(
self._query)
return False
else:
return True
def answer(self):
"""
Check if the question has intent or None
:return: Answer
"""
if self.checkIntent():
# if str(self.CheckJsonEntities()).__contains__("Has No Entities"):
# print("Chatbot : Get Answer From Json Answers")
# else:
self.return_original()
else:
send_message(self.UserID, "not available due to intent")
# print('ChatBot : not available due to intent')
def checkSlots(self):
self.__nluSlots = self._getSlots()
if str(self.__nluSlots).__contains__("No Slots"):
return "No"
datasetSlots = []
for i in range(
len(self.__dataset['intents'][self._getIntent()]
['utterances'])):
for j in range(
len(self.__dataset['intents'][self._getIntent()]
['utterances'][i]['data'])):
check = json.dumps(self.__dataset['intents'][self._getIntent()]
['utterances'][i]['data'][j])
if 'entity' in check:
if str(self.__dataset['intents'][
self._getIntent()]['utterances'][i]['data'][j]
['entity']).__contains__("snips"):
datasetSlots.append(
self.__dataset['intents'][self._getIntent()]
['utterances'][i]['data'][j]['slot_name'])
else:
datasetSlots.append(
self.__dataset['intents'][self._getIntent()]
['utterances'][i]['data'][j]['entity'])
datasetSlots = list(dict.fromkeys(datasetSlots))
for x in datasetSlots:
if 'Val' in x:
datasetSlots.remove(x)
ret = []
for i in range(len(datasetSlots)):
found = False
for j in self.__nluSlots.keys():
if datasetSlots[i] == j:
found = True
break
if not found:
ret.append(datasetSlots[i])
print("Slots Not Available in Question : ",
ret) # el mafroud btalla3 el slots ely mesh mwgooda
return ret
def askForunenteredEntities(self):
slots_needed = self._getSlots()
# print(slots_needed)
slots_missing = self.checkSlots()
if str(slots_missing).__contains__("No"):
return
loopflagi = True
# print(temp3[1]);
for i in range(len(slots_missing)):
loopflagi = False
while not loopflagi:
chatbotques = "Please Enter " + slots_missing[i]
# g = Translator().translate(chatbotques , dest = self.language).text
# print("Chatbot: ",g)
send_message(self.UserID, chatbotques)
# print ("Chatbot : ",chatbotques)
if slots_missing[i] == "ID":
try:
reply = int(input("User: "******"Chatbot: Please Enter a valid ID")
continue
else:
values = []
# print("Chatbot: Possible Values for", slots_missing[i], "are:")
for k in range(
len(self.__dataset['entities'][slots_missing[i]]
['data'])):
values.append(self.__dataset['entities'][
slots_missing[i]]['data'][k]['value'])
# print(values)
send_message(self.UserID, values)
reply = input("User: "******"ID"):
self.__nluSlots[slots_missing[i]] = reply
break
for j in range(
len(self.__dataset['entities'][slots_missing[i]]
['data'])):
try:
check = json.dumps(self.__dataset['entities'][
slots_missing[i]]['data'][j])
# print(check)
except:
continue
if reply in check:
# e3ml 7aga
self.__nluSlots[slots_missing[i]] = reply
loopflagi = True
break
return self.__nluSlots
def return_original(self):
slots_needed = self.askForunenteredEntities()
if slots_needed is None:
self.__interaction.UnAnsweredFactory().__noIntent_insert__(
self._query)
send_message(self.UserID, "I don't have answer for this")
# print("ChatBot : I don't have answer for this")
return
key_list = list(slots_needed.keys())
# print(key_list)
finaldic = dict()
for key in range(len(key_list)):
# print(key_list[key] + "here")
# print(slots_needed[key_list[key]])
# do something
if key_list[key] == "ID":
continue
# print("Key :",key_list[key])
try:
for i in range(
len(self.__dataset['entities'][key_list[key]]
["data"])):
check = json.dumps(
self.__dataset['entities'][key_list[key]]["data"][i])
if slots_needed[key_list[key]] in check:
self.__nluSlots[key_list[key]] = self.__dataset[
'entities'][key_list[key]]["data"][i]["value"]
# print(self.__dataset['entities'][key_list[key]]["data"][i]["value"])
except:
print("Error")
send_message(self.UserID, "Done")
import io
import json
import snips_nlu
from snips_nlu import SnipsNLUEngine
from snips_nlu.default_configs import CONFIG_ES
with io.open('dataset.json') as file:
dataset = json.load(file)
snips_nlu.load_resources("es")
engine = SnipsNLUEngine()
engine.fit(dataset)
parsing = engine.parse(u"Informacion de animales en kitchwua y ingles")
print(json.dumps(parsing, indent=2))
temp=json.dump(parsing, indent=2)
default_engine = SnipsNLUEngine()
#%%
engine = SnipsNLUEngine(config=CONFIG_EN)
with io.open("/Users/ayaali/Documents/Geni_chatbot/Genei/dataset.json") as fil:
dataset = json.load(fil)
#%%
res = []
with open('/Users/ayaali/Documents/Geni_chatbot/Genei/out_new.txt', 'w') as f:
engine.fit(dataset)
seed = 42
engine = SnipsNLUEngine(config=CONFIG_EN, random_state=seed)
engine.fit(dataset)
for ques in mylist:
parsing = engine.parse(ques)
#print(json.dumps(parsng, indent=2))
res.append(parsing)
f.write(json.dumps(res, indent=2))
f.close()
#%%
# with io.open("/Users/ayaali/Documents/Geni_chatbot/Genei/out.txt") as fil:
# res = json.load(fil)
# res
Dicjason={}
for r in range(0,len(res)):
import io
import json
from snips_nlu import SnipsNLUEngine
nlu_engine = SnipsNLUEngine()
with open("./chatbot/data/input.json") as input_file:
input_dict = json.load(input_file)
input = input_dict['input']
with io.open("./chatbot/data/dataset.json") as f:
sample_dataset = json.load(f)
nlu_engine.fit(sample_dataset)
parsing = nlu_engine.parse(u"%s" % input)
file = open("./chatbot/data/output.json", "w")
file.write(json.dumps(parsing, indent=2))
def imprimirmensaje(message):
campo = ""
predicado = ""
bandera1 = False
chatid = message.chat.id
with io.open('dataset.json') as file:
dataset = json.load(file)
engine = SnipsNLUEngine()
engine.fit(dataset)
parsing = engine.parse(unicode(message.text))
entidad = ""
intentName = ""
try:
intentName = parsing["intent"]["intentName"]
try:
enti = parsing["slots"][0]
entidad = enti["rawValue"]
except:
bandera1 = True
if len(intentName) > 0 and len(entidad) > 0:
print("good")
if intentName == "descripcion":
print("si entraaaaaaaaaaaaaaaaaaa")
predicado = "http://usefulinc.com/ns/doap#description"
campo = consultaSparql1(entidad, predicado)
elif intentName == "imagen":
predicado = "http://schema.org/image"
campo = consultaSparql1(entidad, predicado)
campo = ("La imagen de " + entidad + " es: " + campo[0])
elif intentName == "igualEn":
predicado = "http://www.w3.org/2002/07/owl#sameAs"
campo = consultaSparql2(entidad, predicado, "en")
campo = ("La traduccion de " + entidad + " en ingles es: " +
campo[0])
elif intentName == "igualKi":
predicado = "http://www.w3.org/2002/07/owl#sameAs"
campo = consultaSparql2(entidad, predicado, "ki")
campo = ("La traduccion de " + entidad + " en kitchwa es: " +
campo[0])
elif intentName == "nombreCientifico":
predicado = "http://lod.taxonconcept.org/ontology/txn.owl#scientificName"
campo = consultaSparql1(entidad, predicado)
campo = ("El nombre cientifico de " + entidad + " es: " +
campo[0])
mi_bot.reply_to(message, campo)
elif intentName == "allAnimal":
cam = consultaSparql()
campo = ""
for ca in cam:
campo += ca + ", "
mi_bot.reply_to(message, campo)
else:
print(entidad)
print(intentName)
print("bad")
if bandera1:
print("¿Dime de animal deseas saber la descripción?")
#except Exception,e: print str(e)
except:
campo = "No entiendo tu pregunta, puedes preguntarme de nuevo"
mi_bot.reply_to(message, campo)
class DefaultEngine(BaseEngine):
"""
Default Engine module will be used for managing the NLU engine.
"""
def __init__(self, input_object, output_object, engine_param=None):
super(DefaultEngine, self).__init__(input_object, output_object, engine_param)
self.engine_name = "default_engine"
logger.info("Initializing the engine..")
self.engine = SnipsNLUEngine()
# get the path of the dataset
dataset_path = engine_param.get("dataset_path")
self.train_model(dataset_path)
def train_model(self, path):
"""
Train the NLU JSON format by SNIP NLU.
:param str path: path of the dataset.json
"""
with open(path) as dataset_file:
self.engine.fit(json.load(dataset_file))
logger.info("Successfully loading the trained data sets")
def parse(self, request_text):
"""
Parser the given user's text using the the
Snip NLU engine.
"""
engine = self.engine.parse(request_text)
return engine
def response(self, scope):
"""
Get the :class:`Dict` status from NLU or command
execution successfully, the one
response class :mod:`raven.response`
imported.
"""
intent_name = scope["intent"]["intentName"] or "defaultIntent_default"
logger.debug(f"getting the intent class name: {intent_name}")
return import_response_intent(intent_name)(scope=scope)
def command_success_response(self, txObject):
"""
Find if there the given user's text is related
to command request. if then change the scope
intent name as ``commandsIntent_command``.
"""
logger.debug(f"getting the intent class name as: commandsIntent_command")
txObject.update(
{NLUSCOPE: {"intent": {"intentName": "commandsIntent_command"}}}
)
def add(self, layer):
"""
Add the layer function into the engine's
list for executing of the layer function.
:param: :mod:`raven.layer.abstract_layer` layer the object
of function are added as layer into the engine for concurrency
exection.
"""
super(DefaultEngine, self).add(layer)
def go(self, pretty="base.html"):
"""
:param: pretty is the name of the file where the meta data or base line of html
are saved and it is parsed along with return result. Currently base.html and json.html
is taken as parameter.
"""
# super() must be called
super(DefaultEngine, self).go()
if self.break_layer:
# getting the result from the NLP engine.
self.return_object[FOR_OUTPUT] = self.return_object[PROCESSED_INPUT]
self.command_success_response(self.return_object)
else:
self.return_object[NLUSCOPE] = self.parse(
self.return_object[PROCESSED_INPUT]
)
del self.return_object[PROCESSED_INPUT]
# after success of getting return value from the NLU
# or command status the return value are pass into
# response object for template parsing.
_class = self.response(self.return_object[NLUSCOPE])
if self.break_layer:
self.return_object[FOR_OUTPUT] = _class.render(
self.return_object, pretty=pretty
)
else:
self.return_object[FOR_OUTPUT] = _class.render(pretty=pretty)
output_result = super().to_output(self.return_object)
# DEPRECATED:
super().next()
logger.debug(f"running the parser ")
return output_result
#Download resource
#YOU must run this as admin
import subprocess
subprocess.run(["python", "-m", "snips_nlu", "download", "es"])
#----
import io
import json
#Load with io.open("trainingDatasets/multiple.json") as f:
with io.open("trainingDatasets/multiple.json") as f:
sample_dataset = json.load(f)
#Train
from snips_nlu import SnipsNLUEngine
nlu_engine = SnipsNLUEngine()
print(sample_dataset)
nlu_engine.fit(sample_dataset)
#Test
import json
parsing = nlu_engine.parse(
"Creame una máquina virtual con seis cores y 3 gigas de ram en Europa")
print(json.dumps(parsing, indent=2))
print("---------------------")
parsing = nlu_engine.parse("Dame de alta un data lake en Asia")
print(json.dumps(parsing, indent=2))
from __future__ import unicode_literals, print_function
import json
from pathlib import Path
from snips_nlu import SnipsNLUEngine
from snips_nlu.default_configs import CONFIG_EN
SAMPLE_DATASET_PATH = Path(__file__).parent / "sample_dataset.json"
with SAMPLE_DATASET_PATH.open(encoding="utf8") as f:
sample_dataset = json.load(f)
nlu_engine = SnipsNLUEngine(config=CONFIG_EN)
nlu_engine.fit(sample_dataset)
text = "What will be the weather in San Francisco next week?"
parsing = nlu_engine.parse(text)
print(json.dumps(parsing, indent=2))
