def correct(self): sp = SpellCorrectionModel(language='en') module_dir = os.path.dirname(__file__) en = os.path.join(module_dir, 'en_large.pkl') sp.load(en) #module_dir = os.path.dirname(__file__) pathoutF= os.path.join(module_dir, 'gector','Output', 'HTR.txt') f=open(pathoutF,'r') self.text=(sp.spell_correct(f.readline())['spell_corrected_text']) f.close() pathspello=os.path.join(module_dir,'gector','Output','Spello.txt') f=open(pathspello,'w') f.write(self.text) print(self.text) return pathspello
class Corrector: """This function correct the Nepali word based on distance of character. """ def __init__(self): self.this_dir, self.this_file = os.path.split(__file__) self.sp = SpellCorrectionModel(language='hi').load( os.path.join(self.this_dir, "local_dataset", "spell_model.pkl")) def __load_words(self, location): """This function load all the Nepali words in given location. Arguments: location {string} -- [Location where vocabulary is located please be sure to indicate name and extension also.] Returns: [list] -- [List of all Nepali word] """ file = open(os.path.join(self.this_dir, "local_dataset", location), 'r') text = file.read() text = text.split() file.close() return text def corrector(self, word, location='words.txt', number=2, threshold=0.3): """This functon returns 'n' number of correct words. Arguments: word {string} -- [Word you want to check for spelling.] Keyword Arguments: location {str} -- [Location of Vocabulary file with name and extension.] (default: {'../local_dataset/words.txt'}) number {int} -- [Number of close correct words.] (default: {2}) threshold {float} -- [Threshold distance between words enter and predicted word'.] (default: {0.3}) Returns: [list] -- ['n' correct word predictions.] """ word_list = self.__load_words(location) matches = get_close_matches(word, word_list, n=2, cutoff=0.3) return matches def spell_correct(self, text): """ Returns the spell corrected string based on context, phoneme Arguments: text {str} -- Text in Nepali Returns: Dict -- dictionary with correct word, input text and correction text. """ return self.sp.spell_correct(text)
def __init__(self): # self.state = 'start' self.state = 'intent' self.exit = False self.curr_msg = None self.menu = {} self.create_menu_graph() self.browser = 'chrome' # print(self.menu) self.spell_check_model = SpellCorrectionModel(language='en') self.domain_spell_check_model = SpellCorrectionModel(language='en') self.spell_check_model.load('en.pkl') self.domain_spell_check_model.load('custom_model.pkl') with open('data.json', "r") as f: self.db = json.load(f) f.close() # with open('words.json') as f: # self.all_words = json.load(f) # f.close() pk_file = open('final.pkl', 'rb') answers = pickle.load(pk_file) self.answer_detector = Detection(answers=answers, detection_type='answer') intent_answers = [] for each in INTENT_DATA: for i in INTENT_DATA[each]['data']: intent_answers.append( (clean_text(i), INTENT_DATA[each]['number'])) self.intent_detector = Detection(answers=intent_answers, detection_type='intent') port = 465 # For SSL smtp_server = "smtp.gmail.com" self.sender_email = "*****@*****.**" self.receiver_email = "*****@*****.**" password = "******" context = ssl.create_default_context() self.server = smtplib.SMTP_SSL(smtp_server, port, context=context) self.server.login(self.sender_email, password)
def automatedtestforspellcheck(languages): for language in languages: sp = SpellCorrectionModel(language=language) sp.load(f'C:\\Users\\LAP\\{language}.pkl') dir = Path(__file__).resolve().parent test_data = pd.read_csv(dir / f"{language}.csv") def correct(text): try: spello_out = sp.spell_correct(text) return spello_out["spell_corrected_text"] except Exception as e: return e test_data["Output"] = test_data["Sentence with Mistake"].apply(correct) test_data["Clean_original"] = test_data["Original Sentence"].apply( get_clean_text) test_data["Status"] = test_data["Clean_original"] == test_data[ "Output"] out_path = dir / f"{language}_out.csv" results = test_data['Status'].value_counts() print(f"Results for {language} are \n{results}\n") print(f"Dumping results for {language} at {out_path} ...") test_data.to_csv(out_path, index=False)
class ChatBot: # All other functionalities are defined/called in this class. def __init__(self): # self.state = 'start' self.state = 'intent' self.exit = False self.curr_msg = None self.menu = {} self.create_menu_graph() self.browser = 'chrome' # print(self.menu) self.spell_check_model = SpellCorrectionModel(language='en') self.domain_spell_check_model = SpellCorrectionModel(language='en') self.spell_check_model.load('en.pkl') self.domain_spell_check_model.load('custom_model.pkl') with open('data.json', "r") as f: self.db = json.load(f) f.close() # with open('words.json') as f: # self.all_words = json.load(f) # f.close() pk_file = open('final.pkl', 'rb') answers = pickle.load(pk_file) self.answer_detector = Detection(answers=answers, detection_type='answer') intent_answers = [] for each in INTENT_DATA: for i in INTENT_DATA[each]['data']: intent_answers.append( (clean_text(i), INTENT_DATA[each]['number'])) self.intent_detector = Detection(answers=intent_answers, detection_type='intent') port = 465 # For SSL smtp_server = "smtp.gmail.com" self.sender_email = "*****@*****.**" self.receiver_email = "*****@*****.**" password = "******" context = ssl.create_default_context() self.server = smtplib.SMTP_SSL(smtp_server, port, context=context) self.server.login(self.sender_email, password) def greet(self): self.state = 'intent' result = [] if self.db[self.browser]["name"]: greet_msg = "Hello " + self.db[ self.browser]["name"] + "! What brings you back?" else: greet_msg = random.choice(GREETING_PHRASE_1) + " " + random.choice( GREETING_PHRASE_2) result.append((greet_msg, "text")) return result def create_menu_graph(self): data = pd.read_csv("menu.csv", index_col=0, header=0, encoding='utf-8') for row_idx, row in data.iterrows(): # print(row_idx, row['subsections'] if pd.notna(row['subsections']) else False) self.menu[int(row_idx)] = { 'name': '', 'subsections': [], 'data': [] } if pd.notna(row['name']): self.menu[int(row_idx)]['name'] = row['name'] if pd.notna(row['subsections']): self.menu[int( row_idx)]['subsections'] = row['subsections'].split('|') if pd.notna(row['data']): self.menu[int(row_idx)]['data'] = row['data'].split('|') def detect_intent(self, query): intent = self.intent_detector.process_results(query) # print(" detected intent:", intent, "") return intent def detect_answer(self, query): # detect answer from a given question question, answer, score = self.answer_detector.process_results(query) # print("question:", question, " detected answer:", answer, ", score:", score) if score > 0.7: return "<strong>Answer: </strong>" + answer.capitalize() else: return False def handle_sales(self, text): # print("Would you like to get in touch with the F5 team to " # "discover what would be a good fit for your specific use case?") # print("You can type yes/no.") # response = input("user: "******"sales": if response.strip(r'.').lower() in ['yes', 'okay', 'sure']: if self.db[self.browser]["email"] and self.db[ self.browser]["name"]: result.append( ("Customer email id and name are already present.", "text")) result.append(( "Please provide a summary of your " "requirement for our executive to have some context beforehand.", "text")) self.state = "get_summary" return result elif self.db[self.browser]["email"]: self.state = "get_name" result.append( ("Customer email id is already in our database.", "text")) return result.append(("Please provide you name.", "text")) # print("Okay, please provide your work e-mail id.") result.append(("Please provide your work e-mail id.", "text")) self.state = "get_mail" return result elif self.state == "get_mail": email = re.findall( r'[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+', response) if email: # self.send_mail(email) # print("Mail noted. Our sales executive will contact you.") self.add_to_db(email[0], "email") self.state = "get_name" if self.db[self.browser]["name"]: result.append( ("Mail noted and Name is present in our database.", "text")) result.append(( "Last thing, please provide a summary of your " "requirement for our executive to have some context beforehand.", "text")) else: result.append( ("Mail noted. Please provide your name.", "text")) return result else: # self.state = "intent" # print("Email not found.") return result.append( ("Email not found. Lets try that again.", "text")) elif self.state == "get_name": name = response if name: # self.send_mail(email) # print("Mail noted. Our sales executive will contact you.") self.add_to_db(name, "name") result.append(( "Name noted. Last thing, please provide a summary of your " "requirement for our executive to have some context beforehand.", "text")) self.state = "get_summary" return result else: # print("Email not found.") return result.append(("Name not found.", "text")) elif self.state == "get_summary": summary = response self.state = "intent" if summary: # self.send_mail(email) # print("Mail noted. Our sales executive will contact you.") # self.add_to_db(summa, "name") if self.send_mail(summary): result.append( ("Summary noted. Our executive has been notified and " "will contact you soon. Thank you.", "text")) # send email to support staff else: # print("Email not found.") result.append(("Summary not found.", "text")) # result.append(("Okay. Back to main menu.", "text")) # print("Okay. Back to main menu.") self.state = 'intent' result.extend(self.handle_intent(INTENT_DATA['Main menu']['number'])) return result def send_mail(self, summary): message = f"""\ Subject: Customer ticket #147007. We got the following customer ticket through our website chatbot. Please look in to it. Name: {self.db[self.browser]["name"] if self.db[self.browser]["name"] else "Not provided"} email: {self.db[self.browser]["email"]} Summary: {summary} Thanks and regards, Chatbot team. """ # context = ssl.create_default_context() # with smtplib.SMTP_SSL(smtp_server, port, context=context) as server: # server.login(sender_email, password) try: self.server.sendmail(self.sender_email, self.receiver_email, message) except: port = 465 # For SSL smtp_server = "smtp.gmail.com" # belowe are fake mail credentials, using credentials saved in .env files is suggested. Cheers. self.sender_email = "*****@*****.**" self.receiver_email = "*****@*****.**" password = "******" context = ssl.create_default_context() self.server = smtplib.SMTP_SSL(smtp_server, port, context=context) self.server.login(self.sender_email, password) self.server.sendmail(self.sender_email, self.receiver_email, message) return True def get_order(self, sub_sections): with open('data.json', "r") as f: data = json.load(f) f.close() counts = {} if self.browser in data: # print(data) for each in sub_sections: intent = each.strip() if intent in data[self.browser]["history"]: counts[intent] = data[self.browser]["history"][intent] else: counts[intent] = 0 order = [] for i in sorted(counts.items(), key=lambda kv: (kv[1], kv[0]), reverse=True): order.append(int(i[0])) return order def add_to_db(self, command_data, command): with open('data.json', "r") as f: self.db = json.load(f) f.close() file = open("data.json", "r+") file.truncate(0) file.close() command_data = str(command_data).strip() if command == "intent": if self.browser in self.db: if command_data in self.db[self.browser]["history"]: x = self.db[self.browser]["history"][command_data] self.db[self.browser]["history"][command_data] = x + 1 # print("x:", x) # print("updated data:", self.db) else: self.db[self.browser]["history"][command_data] = 1 else: self.db[self.browser] = { "history": {}, "name": "", "email": "" } self.db[self.browser]["history"][command_data] = 1 elif command == "name": self.db[self.browser]["name"] = command_data elif command == "email": self.db[self.browser]["email"] = command_data with open('data.json', "w") as a: json.dump(self.db, a) a.close() def handle_intent(self, intent_num): """ :return: """ data = self.menu[intent_num]['data'] sub_sections = self.menu[intent_num]['subsections'] result = [] if data: for each in data: # result += each.strip() + "" result.append((each.strip(), 'text')) # print("", each.strip()) if sub_sections: order = self.get_order(sub_sections) if order: for i in order: result.append((self.menu[i]['name'], 'button')) else: for each in sub_sections: # result += "->" + self.menu[int(each.strip())]['name'] result.append( (self.menu[int(each.strip())]['name'], 'button')) else: # result += "Would you like to get in touch with the F5 team to " \ # "discover what would be a good fit for your specific use case?" + "" + "You can type yes/no." result.append(( "Would you like to get in touch with the F5 team to " "discover what would be a good fit for your specific use case?", 'text')) result.append(('Yes', 'button')) result.append(('No', 'button')) self.state = "sales" return result def check_domain_context(self, sentence, word): pairs_list = get_context_pairs(sentence.lower().split(), word) for each in pairs_list: a = each in self.domain_spell_check_model.context_model.model_dict b = (each[1], each[0] ) in self.domain_spell_check_model.context_model.model_dict if a or b: return True return False def spell_check(self, text): response = "" final_suggestions_dict = {} global_response = self.spell_check_model.spell_correct(text) domain_response = self.domain_spell_check_model.spell_correct(text) # print("Global_response", global_response) # print("domain_response", domain_response) for each in global_response['correction_dict']: # if each in self.all_words: # continue if each in domain_response['correction_dict']: corrected_domain_each = domain_response['correction_dict'][ each] if self.check_domain_context( domain_response['spell_corrected_text'], corrected_domain_each): final_suggestions_dict[each] = corrected_domain_each else: final_suggestions_dict[each] = global_response[ 'correction_dict'][each] # else: # final_suggestions_dict[each] = global_response['correction_dict'][each] for each in global_response['original_text'].split(): if each in final_suggestions_dict: response += final_suggestions_dict[each] + ' ' else: response += each + ' ' return response.strip() def initialize(self, browser): self.browser = browser if self.browser not in self.db: self.db[self.browser] = {"history": {}, "name": "", "email": ""} with open('data.json', "w") as a: json.dump(self.db, a) a.close() def get_response(self, text, browser, command): # if self.state == 0: # self.greet() result = [] if command == "start": result.extend(self.greet()) else: self.browser = browser self.curr_msg = self.spell_check(text) self.curr_msg = clean_text(self.curr_msg) if self.curr_msg not in STOP_CHAT: if self.state == "intent": if not self.exit: # text = input("user:"******"?]', '', text) # text = re.sub(r'[\.\s]', r' ', text.lower()) # if self.curr_msg not in STOP_CHAT: intent = self.detect_intent(self.curr_msg) answer = self.detect_answer(self.curr_msg) if intent == 6: # support request result.append( ("A support assistant will contact you soon.", "text")) self.state = "sales" result.extend(self.handle_sales("yes")) elif answer and intent: # print("" + answer) self.add_to_db(intent, "intent") result.append((answer + '.', 'text')) result.extend(self.handle_intent(intent)) elif answer: # print("" + str(answer)) # result += self.handle_intent(INTENT_DATA['Main menu']['number']) result.append((answer + '.', 'text')) result.extend( self.handle_intent( INTENT_DATA['Main menu']['number'])) elif intent: self.add_to_db(intent, "intent") # result += self.handle_intent(intent) result.extend(self.handle_intent(intent)) else: result.append( ("Sorry, I did not understand that.", "text")) result.extend( self.handle_intent( INTENT_DATA['Main menu']['number'])) # result += self.handle_intent(INTENT_DATA['Main menu']['number']) elif self.state == "sales": result.extend(self.handle_sales(self.curr_msg)) # result += self.handle_sales(self.curr_msg) elif self.state == "get_mail": result.extend(self.handle_sales(text)) # result.extend(self.handle_intent(INTENT_DATA['Main menu']['number'])) # result += self.handle_sales(text) # result += self.handle_intent(INTENT_DATA['Main menu']['number']) elif self.state == "get_name": result.extend(self.handle_sales(text)) elif self.state == "get_summary": result.extend(self.handle_sales(text)) else: self.exit = True if self.exit: self.exit = False return [("Good bye.", "text")] return result
import os from spello.model import SpellCorrectionModel sp = SpellCorrectionModel(language='en') sp.load(os.getcwd() + '/examples/model.pkl') print(sp.spell_correct('i wnt to ply futbal'))
def __init__(self): self.this_dir, self.this_file = os.path.split(__file__) self.sp = SpellCorrectionModel(language='hi').load( os.path.join(self.this_dir, "local_dataset", "spell_model.pkl"))
import os, csv from spello.model import SpellCorrectionModel sp = SpellCorrectionModel(language='en') sp.train([ 'what is encapsulation', 'what is java', 'abvantage of java', 'why python', 'i want to play football' ]) sp.save(model_save_dir=os.getcwd() + '/examples')