class NLP(): def __init__(self): # load language resources load_resources(u"en") # create NLU Engine self.engine = SnipsNLUEngine(config=CONFIG_EN) # train engine """ with io.open("dataset.json") as f: dataset = json.load(f) print('start training') self.engine.fit(dataset=dataset) print('finished training') self.engine.persist('nlu_engine') """ self.engine = SnipsNLUEngine.from_path('nlu_engine') print('snips engine ready') def parse(self, txt): parsed = self.engine.parse(txt) #print(json.dumps(parsed['slots'], indent=2)) slots = parsed['slots'] try: for slot in slots: output = slot['value']['value'] return True, output except Exception as e: return False, e
class SnipsService(ApiService): def __init__(self, classes, model_path=None, max_api_calls=None, verbose=False): super().__init__(classes, max_api_calls, verbose) load_resources('en') if model_path: self.load_model(model_path) else: self.engine = SnipsNLUEngine(config=CONFIG_EN) def train_model(self, dataset): self.engine.fit(dataset) def train_model_from_file(self, dataset_path): with io.open(dataset_path) as f: self.train_model(json.load(f)) def save_model(self, model_path): self.engine.persist(model_path) def load_model(self, model_path): self.engine = SnipsNLUEngine.from_path(model_path) def predict(self, utterance): result = self.engine.parse(utterance) try: return result['intent']['intentName'] except Exception as e: print('ERR:', e) print('Failed to parse: "{}"'.format(utterance)) print(result) return None
def nluparse(text): load_resources(sample_dataset["language"]) nlu_engine = SnipsNLUEngine(config=config) nlu_engine.fit(sample_dataset) # text = "Show me jobs in LA for today" parsing = nlu_engine.parse(text) return json.dumps(parsing, indent=2)
class ConfigEngine(Engine): def __init__(self): self.engine = None self.config = config def fit(self, dataset): self.engine = SnipsNLUEngine(self.config).fit(dataset) return self def parse(self, text): return self.engine.parse(text)
def conversacion(m): with io.open("training.json") as f: sample_dataset = json.load(f) load_resources("en") nlu_engine = SnipsNLUEngine(config=CONFIG_EN) nlu_engine.fit(sample_dataset) text = (u"" + m.text.lower() + "") listaResultado = nlu_engine.parse(text) return procesarRespuesta(listaResultado)
def make_nlu_model_json(fname): docs = yaml.load_all(stream) ll = [] for doc in docs: i = {} for k, v in doc.items(): i[k] = v ll.append(i) dataset = Dataset.from_yaml_files("en", [ll]) nlu_engine = SnipsNLUEngine(config=CONFIG_EN) nlu_engine = nlu_engine.fit(dataset) text = "Please turn the light on in the kitchen" parsing = nlu_engine.parse(text)
def runEngine(query): with io.open("dataset.json") as f: dataset = json.load(f) load_resources("en") #with io.open("config_en.json") as f: # config = json.load(f) #engine = SnipsNLUEngine(config=config) engine = SnipsNLUEngine(config=CONFIG_EN) engine.fit(dataset) parsing = engine.parse(query) return json.dumps(parsing, indent=2)
def test_default_configs_should_work(self): # Given dataset = deepcopy(WEATHER_DATASET) for language in get_all_languages(): # When config = DEFAULT_CONFIGS.get(language) self.assertIsNotNone(config, "Missing default config for '%s'" % language) dataset[LANGUAGE] = language engine = SnipsNLUEngine(config).fit(dataset) result = engine.parse("Please give me the weather in Paris") # Then intent_name = result[RES_INTENT][RES_INTENT_NAME] self.assertEqual("SearchWeatherForecast", intent_name)
class NLU: def __init__(self): self.sample_dataset = NLU.load_dataset() @staticmethod def load_dataset(): """ Load the sample dataset which will be used to train the snipsnlu NLP engine. :return: """ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) try: # with io.open(str(Path('app', 'static', 'samples', 'sample_dataset.json'))) as fr: # sample_dataset = json.load(fr) # return sample_dataset with open(f'{BASE_DIR}/static/sample/test.json', 'r', encoding='utf-8') as fr: sample_dataset = json.load(fr) return sample_dataset except Exception as e: print("Could not load dataset {}".format(str(e))) def train_engine(self): """ # Setup the snipsnlu NLP engine and pass the training data. :return: """ load_resources("en") self.nlu_engine = SnipsNLUEngine(config=CONFIG_EN) self.nlu_engine.fit(self.sample_dataset) def parse_sentence(self, sentence): """ Get the sentence and parse it to get the result. The sentence is a query made in any natural language(for now we are setting this language as english) and the result is the json string with the parsed help of trained engine and the possible correct prediction of what the query actually meant. :param sentence: :return: """ parsing = self.nlu_engine.parse(sentence) return parsing
def test_sample_configs_should_work(self): # Given dataset = self.sample_dataset for language in get_all_languages(): # When config_file = "config_%s.json" % language config_path = os.path.join(SAMPLES_PATH, "configs", config_file) with io.open(config_path) as f: config = json.load(f) dataset[LANGUAGE] = language engine = SnipsNLUEngine(config).fit(dataset) result = engine.parse("Please give me the weather in Paris") # Then intent_name = result[RES_INTENT][RES_INTENT_NAME] self.assertEqual("sampleGetWeather", intent_name)
def debug_training(dataset_path, config_path=None): if config_path is None: config = NLUEngineConfig() else: with io.open(config_path, "r", encoding="utf8") as f: config = NLUEngineConfig.from_dict(json.load(f)) with io.open(os.path.abspath(dataset_path), "r", encoding="utf8") as f: dataset = json.load(f) engine = SnipsNLUEngine(config).fit(dataset) while True: query = input("Enter a query (type 'q' to quit): ").strip() if isinstance(query, bytes): query = query.decode("utf8") if query == "q": break print(json.dumps(engine.parse(query), indent=2))
def test_keywords_slot_filler_should_work_in_engine(self): # Given dataset_stream = io.StringIO(""" --- type: intent name: SetLightColor utterances: - set the light to [color](blue) in the [room](kitchen) - please make the lights [color](red) in the [room](bathroom)""") dataset = Dataset.from_yaml_files("en", [dataset_stream]).json intent = "SetLightColor" slot_filler_config = { "unit_name": "keyword_slot_filler", "lowercase": True } parser_config = ProbabilisticIntentParserConfig( slot_filler_config=slot_filler_config) engine_config = NLUEngineConfig([parser_config]) engine = SnipsNLUEngine(engine_config).fit(dataset, intent) text = "I want Red lights in the kitchen now" # When res = engine.parse(text) # Then expected_slots = [ custom_slot( unresolved_slot(match_range={ START: 7, END: 10 }, value="Red", entity="color", slot_name="color"), "red"), custom_slot( unresolved_slot(match_range={ START: 25, END: 32 }, value="kitchen", entity="room", slot_name="room")) ] self.assertListEqual(expected_slots, res["slots"])
def imprimirmensaje(message): campo = "" predicado = "" lista = [] chatid = message.chat.id with io.open('dataset.json') as file: dataset = json.load(file) engine = SnipsNLUEngine() engine.fit(dataset) parsing = engine.parse(unicode(message.text)) temp = json.dumps(parsing, indent=2) # try: intentName = parsing["intent"]["intentName"] entidad = "" try: enti = parsing["slots"][0] entidad = enti["rawValue"] except: pass if len(intentName) > 0 and len(entidad) > 0: print("good") if intentName == "descripcion": print("si entraaaaaaaaaaaaaaaaaaa") predicado = "http://usefulinc.com/ns/doap#description" campo = consultaSparql1(entidad, predicado) elif intentName == "imagen": predicado = "http://schema.org/image" campo = consultaSparql1(entidad, predicado) elif intentName == "igualEn": predicado = "http://www.w3.org/2002/07/owl#sameAs" campo = consultaSparql1(entidad, predicado) elif intentName == "igualKi": predicado = "http://www.w3.org/2002/07/owl#sameAs" campo = consultaSparql1(entidad, predicado) elif intentName == "nombreCientifico": predicado = "http://lod.taxonconcept.org/ontology/txn.owl#scientificName" campo = consultaSparql1(entidad, predicado) bot.send_message(chatid, campo) else: print("bad") lista = consultaSparql() bot.send_message(message, "perro")
def debug_training(dataset_path, config_path=None): with Path(dataset_path).open("r", encoding="utf8") as f: dataset = json.load(f) load_resources(dataset["language"]) config = None if config_path is not None: with Path(config_path).open("r", encoding="utf8") as f: config = NLUEngineConfig.from_dict(json.load(f)) engine = SnipsNLUEngine(config).fit(dataset) while True: query = input("Enter a query (type 'q' to quit): ").strip() if isinstance(query, bytes): query = query.decode("utf8") if query == "q": break print(json.dumps(engine.parse(query), indent=2))
def debug_training(dataset_path, config_path=None): with io.open(os.path.abspath(dataset_path), "r", encoding="utf8") as f: dataset = json.load(f) load_resources(dataset["language"]) if config_path is None: config = NLUEngineConfig() else: with io.open(config_path, "r", encoding="utf8") as f: config = NLUEngineConfig.from_dict(json.load(f)) engine = SnipsNLUEngine(config).fit(dataset) while True: query = input("Enter a query (type 'q' to quit): ").strip() if isinstance(query, bytes): query = query.decode("utf8") if query == "q": break print(json.dumps(engine.parse(query), indent=2))
class LanguageParser(pluginmanager.IPlugin, PluginStorage): """ interface to parse input text """ def __init__(self): super(pluginmanager.IPlugin, self).__init__() self._plugins = {} self._pre_train_json = dict() self._pre_train_json['intents'] = {} self._pre_train_json['entities'] = {} self._pre_train_json['language'] = 'en' self.nlu_engine = SnipsNLUEngine() def train(self, plugins): self._generate_pre_train_json(plugins) self.nlu_engine.fit(self._pre_train_json) def _generate_pre_train_json(self, plugins): for plugin in plugins: intent = dict() intent['utterances'] = list() _data = list() _data.append(dict({'text': plugin.get_name()})) intent['utterances'].append(dict({"data": _data})) intent_name = '_'.join(re.findall(r"[\w']+", plugin.get_name())) self._pre_train_json['intents'][intent_name] = intent self._plugins[intent_name] = plugin # handle sub commands (recursive) self._generate_pre_train_json(plugin.get_plugins().values()) def identify_action(self, action): parsed_action = self.nlu_engine.parse(action) print(parsed_action) intent_name = parsed_action['intent']['intentName'] if intent_name not in self._plugins: return None return self._plugins[intent_name]
def test_default_configs_should_work(self): # Given dataset_stream = io.StringIO(""" --- type: intent name: TurnLightOn utterances: - turn on the lights - please switch on the light - switch the light on - can you turn the light on ? - I need you to turn on the lights --- type: intent name: GetWeather utterances: - what is the weather today - What's the weather in tokyo today? - Can you tell me the weather please ? - what is the weather forecast for this weekend""") dataset = Dataset.from_yaml_files("en", [dataset_stream]).json for language in get_all_languages(): # When config = DEFAULT_CONFIGS.get(language) self.assertIsNotNone(config, "Missing default config for '%s'" % language) dataset[LANGUAGE] = language shared = self.get_shared_data(dataset) engine = SnipsNLUEngine(config, **shared).fit(dataset) result = engine.parse("Please give me the weather in Paris") # Then self.assertIsNotNone(result[RES_INTENT]) intent_name = result[RES_INTENT][RES_INTENT_NAME] self.assertEqual("GetWeather", intent_name)
class NLP(): def __init__(self): # load language resources load_resources(u"en") # create NLU Engine self.engine = SnipsNLUEngine(config=CONFIG_EN) # train engine """ with io.open("snips/nlp_db.json") as f: dataset = json.load(f) print('start training') self.engine.fit(dataset=dataset) print('finished training') self.engine.persist('nlu_engine') """ engine_location = rospy.get_param("~engine_location") self.engine = SnipsNLUEngine.from_path(engine_location) print('snips engine ready') def parse(self, txt): parsed = self.engine.parse(txt.decode("utf-8")) #print(json.dumps(parsed['slots'], indent=2)) slots = parsed['slots'] pair = dict() try: for slot in slots: pair[slot['slotName'].encode( "ascii")] = slot['value']['value'].encode("ascii") return True, pair except Exception as e: return False, e
class __impl: __nlu_engine = None def __init__(self): print('Load NLU Engine') print( '-----------------------------------------------------------------' ) try: with io.open("oms_dataset.json") as f: dataset = json.load(f) except: print('I/O error({0}): {1}') sys.exit() load_resources('snips_nlu_en') self.__nlu_engine = SnipsNLUEngine(config=CONFIG_EN) self.__nlu_engine.fit(dataset) self.__nlu_engine.to_byte_array() def parse_text(self, text): return self.__nlu_engine.parse(text)
nlu_engine = SnipsNLUEngine(config=CONFIG_EN) nlu_engine.fit(sample_dataset) nlp = spacy.load('en_core_web_sm') #################Loading Files START###################### f = open("/home/apttus/demo/docs/3PPAgreement/3PPresult.txt", "r", encoding='UTF8') content = f.read() data = content.split("=====================================================") print("\n***************************") for paragraphs in data: paragraph = paragraphs.split("\n\n") blob = TextBlob(paragraph[2]) parsing = nlu_engine.parse(paragraph[2]) print(paragraph[2]) print("\n\n\n") if ('intent' in parsing): #print ("category "+parsing["intent"]["intentName"]) print(json.dumps(parsing, indent=2)) else: print("Null Category") print("\n\n\n") doc = nlp(paragraph[2]) print("Listing Entities ---------------------------------\n") for entity in doc.ents: print(entity.text, entity.label_) print("\n***************************")
class BrewSnips: def __init__(self, json_arg): self.nlu_engine = None self.json_arg = json_arg def simple_text_cleaner(self, query_text): """ Replace, encode, remove unnecessary texts to ensure alignment w/ yaml format. :param query_text: ( String ) Query :return: ( String ) Cleaned-format query """ # Encode into "utf-8" format query_text = query_text.encode(encoding='utf-8').decode("utf-8") # Replace $, %, + into relative words query_text = query_text.replace("$", "") query_text = query_text.replace("%", "percent") query_text = query_text.replace("+", "plus") # Remove punctuations query_text = query_text.translate( str.maketrans('', '', string.punctuation)) # Replace "’" to avoid parsing error query_text = query_text.replace("’", "") return query_text def parse_snips_intent(self): """ Parse original data.json into Snips NLU Engine Training Data in yaml format. Convert into yaml file through command line prompt : 'snips-nlu generate-dataset en input-yaml-file > output-json-file' """ # Get original data.json in DataFrame data_df = DataProcessing( f"{getcwd()}/data_lake/{self.json_arg}").retrieve_process_json() # Get list of Unique Intents intent_list = list(set(data_df["Intent"])) # Load SpaCy NLP Large Corpus spacy_nlp_engine = load('en_core_web_lg') # Init yaml object yaml = ruamel.yaml.YAML() # Set explicit start to True yaml.explicit_start = True # Parse by Intents for intent_name in intent_list: # yes and no are reserved values for yaml file. # To avoid parsing error, "_" is added before the intent name. if intent_name == "yes" or intent_name == "no": intent_dict = {"type": "intent", "name": f"{intent_name}s"} else: intent_dict = {"type": "intent", "name": intent_name} # Init Lists for Slots + Utterances slots_value_list = [] utt_value_list = [] # Subset current Intent Data subset_data = data_df[data_df["Intent"] == intent_name].reset_index(drop=True) # Get current Intent Queries intent_query_words = list(subset_data["Query"]) # Get the 4 grams and convert into a list word_ngrams = (pd.Series(ngrams(intent_query_words, 4))).to_list() # Random sample 80% of each Intent as training phrases for NLU Engine sample_ngrams = sample(word_ngrams, int(len(subset_data) * 0.8)) # Start parsing each queries for phrases in sample_ngrams: # Join phrases back to one single sentence full_text = " ".join(phrases) # Parse Entity of the text through Spacy NLP Engine parse_phrases = spacy_nlp_engine(full_text) # Set slots if len(parse_phrases.ents) > 0: # Get Entity Label and Text, if any for nlp_entity in parse_phrases.ents: entity_label = nlp_entity.label_ entity_text = nlp_entity.text # Construct "slot" for name and entity slot_entities = { "name": entity_label, "entity": entity_label } # Replace text with entity label full_text = full_text.replace( entity_text, f"[{entity_label}]({entity_text})") # Store "utterances" from the ngram utt_value_list.append(full_text) # Store unique "slots" if slot_entities not in slots_value_list: slots_value_list.append(slot_entities) # Set slots in intent dictionary if len(slots_value_list) > 0: intent_dict["slots"] = slots_value_list # Set utterances in intent dictionary if len(utt_value_list) > 0: intent_dict["utterances"] = utt_value_list # If there's no utterances found, use the original ngrams else: intent_dict["utterances"] = [ " ".join(gram) for gram in sample_ngrams ] # Append into output yaml with open(f"{getcwd()}/data_lake/intent_ngram.yaml", "a") as file: yaml.dump(intent_dict, file) def get_nlu_engine(self): """ Get JSON file with our intents and entities tag from part of our input data. Update the Snips NLU Engine. :return: ( Snips NLU Object ) Customized Snips NLU Engine """ # Get parsed intent ngram JSON file with io.open(f"{getcwd()}/data_lake/intent_ngram.json") as f: custom_dataset = json.load(f) # Init Snips NLU Engine self.nlu_engine = SnipsNLUEngine(config=CONFIG_EN) # Fit sample data into NLU Engine for customization self.nlu_engine = self.nlu_engine.fit(custom_dataset) return self.nlu_engine def parse_intent_name_prob(self, text): """ Parse New IntentName and Probability based on customized Snips NLU Engine. :param text: ( String ) Query :return: ( List ) Intent Name and Probability Score """ # Parse queries to determine intent name and probability score parsing = self.nlu_engine.parse(text) # Get intent name intent_name = parsing["intent"]["intentName"] # get the probability intent_prob = parsing["intent"]["probability"] return [intent_name, intent_prob] def brew_intent_score(self): """ Generate new intent name and probability score for each query. Parse them into JSON format and write into a JSON file. """ # Get Customized Snips NLU Engine self.get_nlu_engine() # Get the original json data with io.open(f"{getcwd()}/data_lake/{self.json_arg}") as f: data_df = json.load(f) # Convert list of lists into DataFrame w/ relative columns data_content_df = pd.DataFrame(data_df, columns=["Query", "Intent"]) # Set Intent Similarity Score w/ New Intent data_content_df["NLU_Intent_Score"] = data_content_df[ "Query"].apply(self.parse_intent_name_prob) # Split into individual columns : Intent and Score data_content_df[["NLU_Intent", "NLU_Score"]] = pd.DataFrame( data_content_df["NLU_Intent_Score"].tolist(), index=data_content_df.index) # Drop unused column data_content_df.drop("NLU_Intent_Score", axis=1, inplace=True) # Store all output into feather file write_feather(data_content_df, f"{getcwd()}/data_lake/SnipsNLUData.feather")
from __future__ import unicode_literals, print_function import io import json from snips_nlu import SnipsNLUEngine, load_resources with io.open("sample_dataset.json") as f: sample_dataset = json.load(f) with io.open("configs/config_en.json") as f: config = json.load(f) load_resources(sample_dataset["language"]) nlu_engine = SnipsNLUEngine(config=config) nlu_engine.fit(sample_dataset) text = "What will be the weather in San Francisco next week?" parsing = nlu_engine.parse(text) print(json.dumps(parsing, indent=2))
import io import sys import json from snips_nlu import SnipsNLUEngine import bs4 with io.open("train.json") as f: sample_dataset = json.load(f) nlu_engine = SnipsNLUEngine() nlu_engine.fit(sample_dataset) parsing = nlu_engine.parse(str(" ".join(sys.argv[1:]))) print(json.dumps(parsing, indent=4))
def recognize( text: str, engine: SnipsNLUEngine, slots_dict: typing.Optional[typing.Dict[str, typing.List[str]]] = None, slot_graphs: typing.Optional[typing.Dict[str, nx.DiGraph]] = None, **parse_args, ) -> typing.List[Recognition]: """Recognize intent using Snips NLU.""" result = engine.parse(text, **parse_args) intent_name = result.get("intent", {}).get("intentName") if not intent_name: # Recognition failure return [] slots_dict = slots_dict or {} slot_graphs = slot_graphs or {} recognition = Recognition(text=text, raw_text=text, intent=Intent(name=intent_name, confidence=1.0)) # Replace Snips slot values with Rhasspy slot values (substituted) for slot in result.get("slots", []): slot_name = slot.get("slotName") slot_value_dict = slot.get("value", {}) slot_value = slot_value_dict.get("value") entity = Entity( entity=slot_name, source=slot.get("entity", ""), value=slot_value, raw_value=slot.get("rawValue", slot_value), start=slot["range"]["start"], end=slot["range"]["end"], ) recognition.entities.append(entity) if (not slot_name) or (not slot_value): continue slot_graph = slot_graphs.get(slot_name) if not slot_graph and (slot_name in slots_dict): # Convert slot values to graph slot_graph = rhasspynlu.sentences_to_graph({ slot_name: [ rhasspynlu.jsgf.Sentence.parse(slot_line) for slot_line in slots_dict[slot_name] if slot_line.strip() ] }) slot_graphs[slot_name] = slot_graph entity.tokens = slot_value.split() entity.raw_tokens = list(entity.tokens) if slot_graph: # Pass Snips value through graph slot_recognitions = rhasspynlu.recognize(entity.tokens, slot_graph) if slot_recognitions: # Pull out substituted value and replace in Rhasspy entitiy new_slot_value = slot_recognitions[0].text entity.value = new_slot_value entity.tokens = new_slot_value.split() return [recognition]
class NLU: __connection = Database() __interaction = interaction() __answers = {"NA": "Please Ask your department administrator for this"} # def __Translate_Text (question,x='en'): # translator = Translator() # translated = translator. translate(question , dest=x) # print("Source Language : " + translated.src) # print("Destination Language : " + translated.dest) # print("Origin Text : " + translated.origin) # print("Translated Text : " + translated.text) # return translated.text def __int__(self): self._query = '' self.__result = '' self._intent = '' self._probability = 0 self._slots = {} self.__engine = '' self.__dataset = '' self.__nluSlots = '' self.language = '' self.UserID = '' def EngineMode(self, mode): """ Saving the engine to use the model for every question (Training Part) or Use the model if it already exists (Testing Part) :param mode: Test or Train string :return: Fitted Engine """ if mode == "Train": load_resources('snips_nlu_en') self.__engine = SnipsNLUEngine(config=CONFIG_EN) with io.open("dataset.json") as f: self.__dataset = json.load(f) self.__engine.fit(self.__dataset) #UnComment to save the model #self.__engine.persist("Z:\FCIS-ASU\Semester 8\ChatbotModel") elif mode == "Test": with io.open("dataset.json") as f: self.__dataset = json.load(f) self.__engine = SnipsNLUEngine.from_path( "Z:\FCIS-ASU\Semester 8\ChatbotModel") def setQuery(self, query, user): # self._query , self.language = query self._query = query self.UserID = user self.__excute() self.answer() def __excute(self): parsing = self.__engine.parse(self._query) self.__result = json.loads(json.dumps(parsing, indent=2)) print(self.__result) def _getIntent(self): try: self._intent = self.__result['intent']['intentName'] return self._intent except Exception as e: return 'None' def _getProbability(self): self._probability = self.__result['intent']['probability'] return self._probability #need Handling Exception def _getSlots(self): dic = {} if len(self.__result['slots']) != 0: for x in range(len(self.__result['slots'])): if not re.search("Val", self.__result['slots'][x]['entity']): if str(self.__result['slots'][x]['entity']).__contains__( "snips"): dic[self.__result['slots'][x] ['slotName']] = self.__result['slots'][x]['value'][ 'value'] else: dic[self.__result['slots'][x] ['entity']] = self.__result['slots'][x]['value'][ 'value'] else: return "No Slots" self._slots = dic return self._slots def checkIntent(self): if self._getIntent() == 'None': self.__interaction.UnAnsweredFactory().__noIntent_insert__( self._query) return False else: return True def answer(self): """ Check if the question has intent or None :return: Answer """ if self.checkIntent(): # if str(self.CheckJsonEntities()).__contains__("Has No Entities"): # print("Chatbot : Get Answer From Json Answers") # else: self.return_original() else: send_message(self.UserID, "not available due to intent") # print('ChatBot : not available due to intent') def checkSlots(self): self.__nluSlots = self._getSlots() if str(self.__nluSlots).__contains__("No Slots"): return "No" datasetSlots = [] for i in range( len(self.__dataset['intents'][self._getIntent()] ['utterances'])): for j in range( len(self.__dataset['intents'][self._getIntent()] ['utterances'][i]['data'])): check = json.dumps(self.__dataset['intents'][self._getIntent()] ['utterances'][i]['data'][j]) if 'entity' in check: if str(self.__dataset['intents'][ self._getIntent()]['utterances'][i]['data'][j] ['entity']).__contains__("snips"): datasetSlots.append( self.__dataset['intents'][self._getIntent()] ['utterances'][i]['data'][j]['slot_name']) else: datasetSlots.append( self.__dataset['intents'][self._getIntent()] ['utterances'][i]['data'][j]['entity']) datasetSlots = list(dict.fromkeys(datasetSlots)) for x in datasetSlots: if 'Val' in x: datasetSlots.remove(x) ret = [] for i in range(len(datasetSlots)): found = False for j in self.__nluSlots.keys(): if datasetSlots[i] == j: found = True break if not found: ret.append(datasetSlots[i]) print("Slots Not Available in Question : ", ret) # el mafroud btalla3 el slots ely mesh mwgooda return ret def askForunenteredEntities(self): slots_needed = self._getSlots() # print(slots_needed) slots_missing = self.checkSlots() if str(slots_missing).__contains__("No"): return loopflagi = True # print(temp3[1]); for i in range(len(slots_missing)): loopflagi = False while not loopflagi: chatbotques = "Please Enter " + slots_missing[i] # g = Translator().translate(chatbotques , dest = self.language).text # print("Chatbot: ",g) send_message(self.UserID, chatbotques) # print ("Chatbot : ",chatbotques) if slots_missing[i] == "ID": try: reply = int(input("User: "******"Chatbot: Please Enter a valid ID") continue else: values = [] # print("Chatbot: Possible Values for", slots_missing[i], "are:") for k in range( len(self.__dataset['entities'][slots_missing[i]] ['data'])): values.append(self.__dataset['entities'][ slots_missing[i]]['data'][k]['value']) # print(values) send_message(self.UserID, values) reply = input("User: "******"ID"): self.__nluSlots[slots_missing[i]] = reply break for j in range( len(self.__dataset['entities'][slots_missing[i]] ['data'])): try: check = json.dumps(self.__dataset['entities'][ slots_missing[i]]['data'][j]) # print(check) except: continue if reply in check: # e3ml 7aga self.__nluSlots[slots_missing[i]] = reply loopflagi = True break return self.__nluSlots def return_original(self): slots_needed = self.askForunenteredEntities() if slots_needed is None: self.__interaction.UnAnsweredFactory().__noIntent_insert__( self._query) send_message(self.UserID, "I don't have answer for this") # print("ChatBot : I don't have answer for this") return key_list = list(slots_needed.keys()) # print(key_list) finaldic = dict() for key in range(len(key_list)): # print(key_list[key] + "here") # print(slots_needed[key_list[key]]) # do something if key_list[key] == "ID": continue # print("Key :",key_list[key]) try: for i in range( len(self.__dataset['entities'][key_list[key]] ["data"])): check = json.dumps( self.__dataset['entities'][key_list[key]]["data"][i]) if slots_needed[key_list[key]] in check: self.__nluSlots[key_list[key]] = self.__dataset[ 'entities'][key_list[key]]["data"][i]["value"] # print(self.__dataset['entities'][key_list[key]]["data"][i]["value"]) except: print("Error") send_message(self.UserID, "Done")
import io import json import snips_nlu from snips_nlu import SnipsNLUEngine from snips_nlu.default_configs import CONFIG_ES with io.open('dataset.json') as file: dataset = json.load(file) snips_nlu.load_resources("es") engine = SnipsNLUEngine() engine.fit(dataset) parsing = engine.parse(u"Informacion de animales en kitchwua y ingles") print(json.dumps(parsing, indent=2)) temp=json.dump(parsing, indent=2)
default_engine = SnipsNLUEngine() #%% engine = SnipsNLUEngine(config=CONFIG_EN) with io.open("/Users/ayaali/Documents/Geni_chatbot/Genei/dataset.json") as fil: dataset = json.load(fil) #%% res = [] with open('/Users/ayaali/Documents/Geni_chatbot/Genei/out_new.txt', 'w') as f: engine.fit(dataset) seed = 42 engine = SnipsNLUEngine(config=CONFIG_EN, random_state=seed) engine.fit(dataset) for ques in mylist: parsing = engine.parse(ques) #print(json.dumps(parsng, indent=2)) res.append(parsing) f.write(json.dumps(res, indent=2)) f.close() #%% # with io.open("/Users/ayaali/Documents/Geni_chatbot/Genei/out.txt") as fil: # res = json.load(fil) # res Dicjason={} for r in range(0,len(res)):
import io import json from snips_nlu import SnipsNLUEngine nlu_engine = SnipsNLUEngine() with open("./chatbot/data/input.json") as input_file: input_dict = json.load(input_file) input = input_dict['input'] with io.open("./chatbot/data/dataset.json") as f: sample_dataset = json.load(f) nlu_engine.fit(sample_dataset) parsing = nlu_engine.parse(u"%s" % input) file = open("./chatbot/data/output.json", "w") file.write(json.dumps(parsing, indent=2))
def imprimirmensaje(message): campo = "" predicado = "" bandera1 = False chatid = message.chat.id with io.open('dataset.json') as file: dataset = json.load(file) engine = SnipsNLUEngine() engine.fit(dataset) parsing = engine.parse(unicode(message.text)) entidad = "" intentName = "" try: intentName = parsing["intent"]["intentName"] try: enti = parsing["slots"][0] entidad = enti["rawValue"] except: bandera1 = True if len(intentName) > 0 and len(entidad) > 0: print("good") if intentName == "descripcion": print("si entraaaaaaaaaaaaaaaaaaa") predicado = "http://usefulinc.com/ns/doap#description" campo = consultaSparql1(entidad, predicado) elif intentName == "imagen": predicado = "http://schema.org/image" campo = consultaSparql1(entidad, predicado) campo = ("La imagen de " + entidad + " es: " + campo[0]) elif intentName == "igualEn": predicado = "http://www.w3.org/2002/07/owl#sameAs" campo = consultaSparql2(entidad, predicado, "en") campo = ("La traduccion de " + entidad + " en ingles es: " + campo[0]) elif intentName == "igualKi": predicado = "http://www.w3.org/2002/07/owl#sameAs" campo = consultaSparql2(entidad, predicado, "ki") campo = ("La traduccion de " + entidad + " en kitchwa es: " + campo[0]) elif intentName == "nombreCientifico": predicado = "http://lod.taxonconcept.org/ontology/txn.owl#scientificName" campo = consultaSparql1(entidad, predicado) campo = ("El nombre cientifico de " + entidad + " es: " + campo[0]) mi_bot.reply_to(message, campo) elif intentName == "allAnimal": cam = consultaSparql() campo = "" for ca in cam: campo += ca + ", " mi_bot.reply_to(message, campo) else: print(entidad) print(intentName) print("bad") if bandera1: print("¿Dime de animal deseas saber la descripción?") #except Exception,e: print str(e) except: campo = "No entiendo tu pregunta, puedes preguntarme de nuevo" mi_bot.reply_to(message, campo)
class DefaultEngine(BaseEngine): """ Default Engine module will be used for managing the NLU engine. """ def __init__(self, input_object, output_object, engine_param=None): super(DefaultEngine, self).__init__(input_object, output_object, engine_param) self.engine_name = "default_engine" logger.info("Initializing the engine..") self.engine = SnipsNLUEngine() # get the path of the dataset dataset_path = engine_param.get("dataset_path") self.train_model(dataset_path) def train_model(self, path): """ Train the NLU JSON format by SNIP NLU. :param str path: path of the dataset.json """ with open(path) as dataset_file: self.engine.fit(json.load(dataset_file)) logger.info("Successfully loading the trained data sets") def parse(self, request_text): """ Parser the given user's text using the the Snip NLU engine. """ engine = self.engine.parse(request_text) return engine def response(self, scope): """ Get the :class:`Dict` status from NLU or command execution successfully, the one response class :mod:`raven.response` imported. """ intent_name = scope["intent"]["intentName"] or "defaultIntent_default" logger.debug(f"getting the intent class name: {intent_name}") return import_response_intent(intent_name)(scope=scope) def command_success_response(self, txObject): """ Find if there the given user's text is related to command request. if then change the scope intent name as ``commandsIntent_command``. """ logger.debug(f"getting the intent class name as: commandsIntent_command") txObject.update( {NLUSCOPE: {"intent": {"intentName": "commandsIntent_command"}}} ) def add(self, layer): """ Add the layer function into the engine's list for executing of the layer function. :param: :mod:`raven.layer.abstract_layer` layer the object of function are added as layer into the engine for concurrency exection. """ super(DefaultEngine, self).add(layer) def go(self, pretty="base.html"): """ :param: pretty is the name of the file where the meta data or base line of html are saved and it is parsed along with return result. Currently base.html and json.html is taken as parameter. """ # super() must be called super(DefaultEngine, self).go() if self.break_layer: # getting the result from the NLP engine. self.return_object[FOR_OUTPUT] = self.return_object[PROCESSED_INPUT] self.command_success_response(self.return_object) else: self.return_object[NLUSCOPE] = self.parse( self.return_object[PROCESSED_INPUT] ) del self.return_object[PROCESSED_INPUT] # after success of getting return value from the NLU # or command status the return value are pass into # response object for template parsing. _class = self.response(self.return_object[NLUSCOPE]) if self.break_layer: self.return_object[FOR_OUTPUT] = _class.render( self.return_object, pretty=pretty ) else: self.return_object[FOR_OUTPUT] = _class.render(pretty=pretty) output_result = super().to_output(self.return_object) # DEPRECATED: super().next() logger.debug(f"running the parser ") return output_result
#Download resource #YOU must run this as admin import subprocess subprocess.run(["python", "-m", "snips_nlu", "download", "es"]) #---- import io import json #Load with io.open("trainingDatasets/multiple.json") as f: with io.open("trainingDatasets/multiple.json") as f: sample_dataset = json.load(f) #Train from snips_nlu import SnipsNLUEngine nlu_engine = SnipsNLUEngine() print(sample_dataset) nlu_engine.fit(sample_dataset) #Test import json parsing = nlu_engine.parse( "Creame una máquina virtual con seis cores y 3 gigas de ram en Europa") print(json.dumps(parsing, indent=2)) print("---------------------") parsing = nlu_engine.parse("Dame de alta un data lake en Asia") print(json.dumps(parsing, indent=2))
from __future__ import unicode_literals, print_function import json from pathlib import Path from snips_nlu import SnipsNLUEngine from snips_nlu.default_configs import CONFIG_EN SAMPLE_DATASET_PATH = Path(__file__).parent / "sample_dataset.json" with SAMPLE_DATASET_PATH.open(encoding="utf8") as f: sample_dataset = json.load(f) nlu_engine = SnipsNLUEngine(config=CONFIG_EN) nlu_engine.fit(sample_dataset) text = "What will be the weather in San Francisco next week?" parsing = nlu_engine.parse(text) print(json.dumps(parsing, indent=2))