def getTicketFlag(): logging.info('getTicketFlag : ') cust_id = '' received_data = request.json try: cust_id = received_data['currentAccount']['subdomain'] except KeyError as err: logging.error(err) cust_id = 'default' cust = getCustomerModel().authenticate(cust_id.strip().lower(), newflag=False) if cust == None: cust_id = 'default' else: cust_id = cust['cust_name'] logging.info('Customer Id : ' + str(cust_id)) if cust_id == 'default': status_flag = False else: status_flag = getCustomerModel().getTicketFlag(cust_id) resp = {} resp["Ticket_Flag"] = status_flag json_resp = json.dumps(resp) return json_resp
def doregister(): cust_id = request.args.get('cust_id', None) lang_type = request.args.get('lang_type', None) email_id = request.args.get('email_id', None) password_ = request.args.get('password', None) #emailsender = EmailSender() #emailsender.send() if cust_id == None or cust_id == '': logging.error('\'' + cust_id + '\' is invalid customer subdomain. ') return render_template("register.html", error_msg='Error: Invalid User Organization') cust_id = cust_id.strip().lower() intenteng = IntentExtractor() cust = getCustomerModel().authenticate(cust_id) if cust == None: cust = getCustomerModel().create(cust_id, language=lang_type, email_id=email_id, password = password_, newflag=True, retrainflag=True, ticketflag = True, done=True) if cust: tickets_learner().import_responsedata(cust['cust_name'], cust['language']) tickets_learner().import_trainingdata(cust['cust_name'], cust['language']) intenteng.prepareTrainingData_nospacy(cust['cust_name']) intenteng.startTrainingProcess(cust['cust_name']) return redirect(url_for('.list', cust_id=cust_id)) else: return redirect(url_for('.list', cust_id=cust_id)) logging.error('\'' + cust_id + '\' is invalid customer subdomain. ') return render_template("register.html", error_msg='Error: Invalid User Organization')
def extractFeedbackData_default(self, src_cust_id='default'): logging.info ('extractFeedbackData_default : Started : ' + str(src_cust_id)) trainlog = get_model() traindata = getTrainingModel() next_page_token = 0 token = None cust_list, next_page_token2 = getCustomerModel().list(done=True) while next_page_token != None: intent_logs, next_page_token = trainlog.list(log_type='feedback', cursor=token, cust_id=src_cust_id, done=True) token = next_page_token for intent_log in intent_logs: intents_data = intent_log["json_data"] intents_data_json = json.loads(intents_data) selected_response_id = intents_data_json["selected_response_id"] cust_id = intents_data_json["ticket_data"]['currentAccount']['subdomain'] id = intents_data_json["ticket_data"]['id'] for cust_id_x in cust_list: if cust_id_x['cust_name'] == cust_id: train_data = traindata.read(id, cust_id=cust_id) response_data = getResponseModel().read(selected_response_id, cust_id=cust_id) if train_data != None and response_data != None: traindata.update(train_data["tags"], train_data["query"], train_data["response"], query_category=train_data['query_category'], resp_category=response_data['res_category'], feedback_flag=True, done=True, id=train_data['id'], cust_id=cust_id) trainlog.delete(intent_log['id'], cust_id=src_cust_id) print('Updating Feedback : ' , id, cust_id) logging.info ('extractFeedbackData_default : Completed : ' + str(src_cust_id))
def prepareTrainingData(self, cust_id): logging.info("prepareTrainingData : Started " + str(cust_id)) self.X, self.y = [], [] tickets_learn = tickets_learner() ticket_data = tickets_learn.getTrainingData(cust_id=cust_id) from agentapp.UtilityClass_spacy import UtilityClass_spacy utilspace = UtilityClass_spacy() lang = getCustomerModel().getLanguage(cust_id) xX = [] yY = [] for linestms in ticket_data: for linestm in linestms: strx = str(linestm['tags'] + ' . ' + linestm['query']).strip() strx = utilspace.preprocessText(strx, lang=lang, ner=True) strx = str(self.utilclass.cleanData(strx, lang=lang, lowercase=True, remove_stops=True, tag_remove=True)) if (strx != ''): xX.append(strx.strip().split()) yY.append(str(linestm['resp_category']).strip()) self.X = xX self.y = yY self.X, self.y = np.array(self.X, dtype=object), np.array(self.y, dtype=object) logging.info ("Total Training Examples : %s" % len(self.y)) logging.info ("prepareTrainingData : Completed " + str(cust_id)) return
def extractTicketData_default(self, src_cust_id='default'): logging.info ('extractTicketData_default : Started :' + str(src_cust_id)) trainlog = get_model() traindata = getTrainingModel() next_page_token = 0 token = None cust_list, next_page_token2 = getCustomerModel().list(done=True) while next_page_token != None: ticket_logs, next_page_token = trainlog.list(log_type='tickets', cursor=token, cust_id=src_cust_id, done=True) token = next_page_token for ticket_log in ticket_logs: tickets_data = ticket_log["json_data"] tickets_data_json = json.loads(tickets_data) for ticket_data in tickets_data_json['tickets']: description = ticket_data['description'] subject = ticket_data['subject'] tags = ', '.join(ticket_data['tags']) url = ticket_data['url'] id = ticket_data['id'] for cust_id_x in cust_list: if cust_id_x['cust_name'] in url: traindata.create(tags, str(subject + ' . ' + description), '', done=False, id=id, cust_id=cust_id_x['cust_name']) trainlog.delete(ticket_log['id'], cust_id=src_cust_id) print('Creating for Ticket Id : ' , ticket_data['id'], cust_id_x['cust_name']) logging.info ('extractTicketData_default : Completed : ' + str(src_cust_id))
def edit(id): cust_id = request.args.get('cust_id', None) if cust_id == None or cust_id == '': return render_template("login.html") cust_id = cust_id.strip().lower() cust = getCustomerModel().authenticate(cust_id) if cust == None: logging.error('\'' + cust_id + '\' is invalid customer Organization. ') return render_template("login.html", error_msg='Error: Invalid User Organization') book = getResponseModel().read(id, cust_id) if request.method == 'POST': data = request.form.to_dict(flat=True) orgdata = getResponseModel().read(id, cust_id=cust_id) book = getResponseModel().update(data['resp_name'], orgdata['res_category'], data['response_text'], data['tags'], orgdata['resp_tags'], modifiedflag=True, done=True, id=id, cust_id=cust_id) print (orgdata['resp_name']) ''' trainlist = getTrainingModel().list_by_respcategory(orgdata['resp_name'], cust_id=cust_id) for resp in trainlist: if (resp != None) and (len(resp) > 0) : for resp_item in resp: getTrainingModel().update(data['tags'], resp_item['query'], resp_item['response'], resp_item['query_category'], resp_category=resp_item['resp_category'], done=True, id=resp_item['id'], cust_id=cust_id) ''' return redirect(url_for('.view', cust_id=cust_id, id=book['id'])) return render_template("form.html", cust_id=cust_id, action="Edit", book=book)
def extractIntentData_default(self, src_cust_id='default'): logging.info ('extractIntentData_default : Started : ' + str(src_cust_id)) trainlog = get_model() traindata = getTrainingModel() next_page_token = 0 token = None cust_list, next_page_token2 = getCustomerModel().list(done=True) while next_page_token != None: intent_logs, next_page_token = trainlog.list(log_type='intent', cursor=token, cust_id=src_cust_id, done=True) token = next_page_token for intent_log in intent_logs: intents_data = intent_log["json_data"] intents_data_json = json.loads(intents_data) description = intents_data_json['description'] subject = intents_data_json['subject'] tags = ', '.join(intents_data_json['requester']['tags']) cust_id = intents_data_json['currentAccount']['subdomain'] id = intents_data_json['id'] response = '' if len(intents_data_json['comments']) > 0: response = intents_data_json['comments'][0]['value'] response = cleanhtml (response) for cust_id_x in cust_list: if cust_id_x['cust_name'] == cust_id: traindata.create(tags, str(subject + ' . ' + description), response, id=id, done=False, cust_id=cust_id) trainlog.delete(intent_log['id'], cust_id=src_cust_id) print('Creating for Intent : ' , intents_data_json['id'], cust_id) logging.info ('extractIntentData_default : Completed : ' + str(src_cust_id))
def doretrain(): cust_id = request.args.get('cust_id', None) if cust_id == None or cust_id == '': logging.error('\'' + cust_id + '\' is invalid customer subdomain. ') return render_template("register.html", error_msg='Error: Invalid User Organization') cust_id = cust_id.strip().lower() cust = getCustomerModel().authenticate(cust_id) if cust == None: logging.error('\'' + cust_id + '\' is invalid customer Organization. ') return render_template("login.html", error_msg='Error: Invalid User Organization') else: getCustomerModel().addretraining(cust_id) #return render_template("list.html", error_msg='Thanks for submitting data. Soon, we will retain the model and notify you.', cust_id=cust_id) return redirect(url_for('.list', error_msg='Thanks for submitting data. Soon, we will retain the model and notify you.', cust_id=cust_id))
def auth(): cust_id = request.args.get('cust_id', None) if cust_id == None or cust_id == '': return render_template("login.html", error_msg='Error: Invalid User Organization') cust_id = cust_id.strip().lower() cust = getCustomerModel().authenticate(cust_id) if cust == None: logging.error('\'' + cust_id + '\' is invalid customer Organization. ') return render_template("login.html", error_msg='Error: Invalid User Organization') return redirect(url_for('.list', cust_id=cust_id))
def route(): cust_id = request.args.get('cust_id', None) if cust_id == None or cust_id == '': return render_template("register.html", error_msg='Error: Invalid User Organization') cust_id = cust_id.strip().lower() cust = getCustomerModel().authenticate(cust_id) if cust == None: return render_template("register.html", error_msg='Error: Invalid User Organization') return redirect(url_for('.list', cust_id=cust_id))
def view(id): cust_id = request.args.get('cust_id', None) if cust_id == None or cust_id == '': return render_template("login.html") cust_id = cust_id.strip().lower() cust = getCustomerModel().authenticate(cust_id) if cust == None: logging.error('\'' + cust_id + '\' is invalid customer Organization. ') return render_template("login.html") book = getResponseModel().read(id, cust_id) return render_template("view.html", cust_id=cust_id, book=book)
def getPredictedIntent(self, textinput, cust_id): logging.info("getPredictedIntent : Started " + str(cust_id)) pickle_out = self.storage.get_bucket(cust_id) self.etree_w2v_tfidf = pickle.loads(pickle_out) self.test_X = [] lang = getCustomerModel().getLanguage(cust_id) strx = str(self.utilclass.cleanData(textinput, lang=lang, lowercase=True, remove_stops=True, tag_remove=True)) self.test_X.append(strx.strip().split()) self.predicted = [] try: self.predicted = self.etree_w2v_tfidf.predict(self.test_X) except ValueError as err: logging.error('getPredictedIntent : ' + str(err)) logging.info("getPredictedIntent : Completed " + str(cust_id)) return self.predicted
def getPredictedIntent_list(self, X_in, cust_id): logging.info("getPredictedIntent_list : Started " + str(cust_id)) pickle_out = self.storage.get_bucket(cust_id) self.etree_w2v_tfidf = pickle.loads(pickle_out) lang = getCustomerModel().getLanguage(cust_id) X_in = X_in.apply(lambda x : self.utilclass.cleanData(x, lang=lang, lowercase=True, remove_stops=True, tag_remove=True).strip().split()) predicted_list= [] try: predicted_list = self.etree_w2v_tfidf.predict(X_in) except ValueError as err: logging.error('getPredictedIntent_list : ' + str(err)) predict_df = pd.Series(predicted_list) logging.info("getPredictedIntent_list : Completed " + str(cust_id)) return predict_df
def startTrainLogPrediction(self, cust_id): logging.info ('startTrainLogPrediction : Started : ' + str(cust_id)) traindata = getTrainingModel() next_page_token = 0 token = None lang = getCustomerModel().getLanguage(cust_id) while next_page_token != None: training_logs, next_page_token = traindata.list(cursor=token, cust_id=cust_id, feedback_flag=False, done=False) token = next_page_token for training_log in training_logs: strx = training_log['query'] + ' . ' + training_log['tags'] strx = self.utilclass.cleanData(strx, lang=lang, lowercase=True, remove_stops=True, tag_remove=True) predicted = self.getPredictedIntent(strx, cust_id) if len(predicted) < 1: predicted = ['Default'] traindata.update(training_log['tags'], training_log['query'], training_log['response'], query_category=training_log['query_category'], done=True, id = training_log['id'], resp_category=predicted[0], cust_id=cust_id) print ('processing training data :', training_log['id']) logging.info ('startTrainLogPrediction : Completed : ' + cust_id)
def getPredictedIntent_prob(self, X_in, cust_id): logging.info("getPredictedIntent_prob : Started " + str(cust_id)) if self.model == None: logging.info('Cant process as no Training ') return lang = getCustomerModel().getLanguage(cust_id) X_in = X_in.apply(lambda x: self.utilclass.cleanData( x, lang=lang, lowercase=True, remove_stops=True, tag_remove=True). strip().split()) try: predicted_list = self.etree_w2v_tfidf.predict(X_in) predicted_prob = self.etree_w2v_tfidf.predict_proba(X_in) except ValueError as err: logging.error('getPredictedIntent_prob : ' + str(err)) intent_predict_df = pd.DataFrame(predicted_list, columns=['predicted_list']) #intent_predict_df ['predicted_prob'] = pd.Series(predicted_prob) #print (predicted_prob) predict_prob = [] predict_class = [] for items in predicted_prob: predict_df = pd.DataFrame({ 'Class': self.etree_w2v_tfidf.classes_, 'Prob': items }) predict_prob.append(predict_df.loc[predict_df['Prob'].idxmax(), 'Prob']) predict_class.append(predict_df.loc[predict_df['Prob'].idxmax(), 'Class']) intent_predict_df['predict_class'] = pd.Series(predict_class) intent_predict_df['predict_prob'] = pd.Series(predict_prob) #print (intent_predict_df) logging.info("getPredictedIntent_list : Completed " + str(cust_id)) return intent_predict_df['predict_class'], intent_predict_df[ 'predict_prob']
def delete(id): cust_id = request.args.get('cust_id', None) if cust_id == None or cust_id == '': return render_template("login.html") cust_id = cust_id.strip().lower() cust = getCustomerModel().authenticate(cust_id) if cust == None: logging.error('\'' + cust_id + '\' is invalid customer Organization. ') return render_template("login.html", error_msg='Error: Invalid User Organization') dataitm = getResponseModel().read(id, cust_id) ''' trainlist = getTrainingModel().list_by_respcategory(dataitm['resp_name'], cust_id=cust_id, done=True) for resp in trainlist: if (resp != None) and (len(resp) > 0) : for resp_item in resp: getTrainingModel().delete(resp_item['id'], cust_id=cust_id) ''' getResponseModel().delete(id, cust_id) return redirect(url_for('.list', cust_id=cust_id))
def getPredictedIntent_list(self, X_in, cust_id): logging.info("getPredictedIntent_list : Started " + str(cust_id)) if self.model == None: logging.info('Cant process as no Training ') return lang = getCustomerModel().getLanguage(cust_id) X_in = X_in.apply(lambda x: self.utilclass.cleanData( x, lang=lang, lowercase=True, remove_stops=True, tag_remove=True). strip().split()) predicted_list = [] try: predicted_list = self.etree_w2v_tfidf.predict(X_in) except ValueError as err: logging.error('getPredictedIntent_list : ' + str(err)) except AttributeError as err: logging.error('getPredictedIntent_list : ' + str(err)) predict_df = pd.Series(predicted_list) logging.info("getPredictedIntent_list : Completed " + str(cust_id)) return predict_df
def uploadFeedback(): logging.info('uploadFeedback : ') received_data = request.json cust_id = '' try: cust_id = received_data['ticket_data']['currentAccount']['subdomain'] except KeyError as err: logging.info(err) cust_id = 'default' cust = getCustomerModel().authenticate(cust_id.strip().lower(), newflag=False) if cust == None: cust_id = 'default' else: cust_id = cust['cust_name'] logging.info('Customer Id : ' + str(cust_id)) get_model().create('feedback', json.dumps(request.json), done=True, cust_id=cust_id) return '200'
def add(): cust_id = request.args.get('cust_id', None) if cust_id == None or cust_id == '': return render_template("login.html") cust_id = cust_id.strip().lower() cust = getCustomerModel().authenticate(cust_id) if cust == None: logging.error('\'' + cust_id + '\' is invalid customer Organization. ') return render_template("login.html") if request.method == 'POST': data = request.form.to_dict(flat=True) book = getResponseModel().create(data['resp_name'], data['resp_name'].strip().lower().replace(" ", "_"), data['response_text'], data['tags'], data['tags'], modifiedflag=True, done=True, id=None, cust_id=cust_id) traindata = getTrainingModel().create(data['tags'], '', data['response_text'], query_category='', resp_category=data['resp_name'].strip().lower().replace(" ", "_"), done=True, id=None, cust_id=cust_id) return redirect(url_for('.view', cust_id=cust_id, id=book['id'])) return render_template("form.html", cust_id=cust_id, action="Add", book={})
def prepareTestingData(self, cust_id): logging.info("prepareTestingData : Started " + str(cust_id)) self.test_X, self.test_y = [], [] tickets_learn = tickets_learner() ticket_data = tickets_learn.getTrainingData(cust_id=cust_id) lang = getCustomerModel().getLanguage(cust_id) xX = [] yY = [] for linestms in ticket_data: for linestm in linestms: logging.debug (str(linestm['tags'] + ', ' + linestm['query']) + " => " + linestm['response']) strx = str(linestm['tags'] + ', ' + linestm['query']) strx = self.utilclass.cleanData(strx, lang=lang, lowercase=True, remove_stops=True, tag_remove=True) xX.append(strx.strip().split()) yY.append(linestm['resp_category'].strip()) self.test_X = xX self.test_y = yY self.test_X, self.test_y = np.array(self.test_X, dtype=object), np.array(self.test_y, dtype=object) logging.info ("Total Testing Examples : %s" % len(self.test_y)) logging.info("prepareTestingData : Completed " + str(cust_id)) return
def getPredictedIntent(self, textinput, cust_id): logging.info("getPredictedIntent : Started " + str(cust_id)) if self.model == None: logging.info('Cant process as no Training ') return self.test_X = [] lang = getCustomerModel().getLanguage(cust_id) strx = self.utilclass.cleanData(textinput, lang=lang, lowercase=True, remove_stops=True, tag_remove=True) self.test_X.append(strx.strip().split()) self.predicted = [] try: self.predicted = self.etree_w2v_tfidf.predict(self.test_X) except ValueError as err: logging.error('getPredictedIntent : ' + str(err)) if len(self.predicted) < 1: self.predicted.append('default') logging.info("getPredictedIntent : Completed " + str(cust_id)) return self.predicted
def uploadArticles(): logging.info('uploadArticles : ') cust_id = '' received_data = request.json #print (received_data) try: cust_id = received_data['ticket_data']['currentAccount']['subdomain'] except KeyError as err: logging.error(err) return '200' cust = getCustomerModel().authenticate(cust_id.strip().lower(), newflag=False) if cust == None: cust_id = 'default' else: cust_id = cust['cust_name'] logging.info('Customer Id : ' + str(cust_id)) response_struct = [] article_data = received_data['article_data'] respdata = getResponseModel() for items in article_data: response_text = str(items['body']) resp_name = items['title'].split()[:5] resp_name = ' '.join(resp_name).title().replace(" ", "_") utils_class = UtilityClass() response_text = utils_class.cleanhtml(response_text) if len(response_text) > 600: response_text = str ('Please find response for your query at following link. \n ' + items['html_url']) response_struct.append({'id' : items['id'], 'resp_name': resp_name, 'res_category': str(cust_id + '_Article_' + str(items['id'])), 'response_text' : response_text, 'tags' : items['title'], 'modifiedflag': False, 'defaultflag' : False, 'resp_tags' : '', 'created': datetime.datetime.utcnow(), 'done': True}) if (len (response_struct) > 1): response_pd = pd.DataFrame(response_struct) respdata.batchUpdate(response_pd, cust_id) return '200'
def predictIntent(): logging.info('predictIntent : ') intent_input = '' cust_id = '' intenteng = IntentExtractor() ticketLearner = tickets_learner() utilclass = UtilityClass() received_data = request.json try: cust_id = received_data['currentAccount']['subdomain'] except KeyError as err: logging.error(err) cust_id = 'default' cust = getCustomerModel().authenticate(cust_id.strip().lower(), newflag=False) if cust == None: cust_id = 'default' else: cust_id = cust['cust_name'] logging.info('Customer Id : ' + str(cust_id)) get_model().create('intent', json.dumps(request.json), done=True, cust_id=cust_id) len_comment = len(received_data['comments']) if ((len_comment > 0) and (received_data['requester']['email'] == received_data['comments'][0]['author']['email'])): intent_input = utilclass.cleanhtml(received_data['comments'][0]['value']) else: intent_input = utilclass.cleanhtml(received_data['description'] + '. ' + received_data['subject']) predicted_intent = intenteng.getIntentForText(intent_input, cust_id) formatted_resp = ticketLearner.formatOutput(predicted_intent, cust_id) logging.info('\'' + str(intent_input) + '\' >> ' + str(formatted_resp)) json_resp = json.dumps(formatted_resp) #get_model().create('response', json_resp, done=True, cust_id=cust_id) return json_resp
def list(): token = request.args.get('page_token', None) cust_id = request.args.get('cust_id', None) error_msg = request.args.get('error_msg', None) if error_msg == None: error_msg = '' if cust_id == None or cust_id == '': return render_template("login.html") cust_id = cust_id.strip().lower() cust = getCustomerModel().authenticate(cust_id) if cust == None: logging.error('\'' + cust_id + '\' is invalid customer Organization. ') return render_template("login.html") if token: token = token.encode('utf-8') books, next_page_token = getResponseModel().list(cursor=token, cust_id=cust_id, done=True, modifiedflag=None, defaultflag=None) return render_template( "list.html", cust_id=cust_id, error_msg=error_msg, books=books, next_page_token=next_page_token)
def copyDefaultTrainingLog(self): logging.info ('copyDefaultTrainingLog : Started :' ) trainlog = get_model() traindata = getTrainingModel() next_page_token = 0 token = None cust_list, next_page_token2 = getCustomerModel().list(done=True) while next_page_token != None: ticket_logs, next_page_token = trainlog.list(cursor=token, cust_id='default', done=True) token = next_page_token for ticket_log in ticket_logs: tickets_data = ticket_log["json_data"] if (ticket_log['type'] == 'response'): continue tickets_data_json = json.loads(tickets_data) tik_data1 = None try : tik_data1 = tickets_data_json['ticket_data'] except KeyError as err: tik_data1 = None if tik_data1 == None: # Intent / Old TIcket tik_data2 = None try : tik_data2 = tickets_data_json['currentAccount']['subdomain'] except KeyError as err: tik_data2 = None if tik_data2 != None: # Intent cust_id = tickets_data_json['currentAccount']['subdomain'] for cust_id_x in cust_list: if cust_id_x['cust_name'] == cust_id: trainlog.create(ticket_log['type'], ticket_log['json_data'], created=ticket_log['created'], done=True, cust_id=cust_id) trainlog.delete(id=ticket_log['id'], cust_id='default') print ('Copying Intent data: ' , ticket_log['id'], cust_id) continue tik_data3 = None try : tik_data3 = tickets_data_json['tickets'][0]['url'] except KeyError as err: tik_data3 = None if tik_data3 != None : # Old Ticket url = tickets_data_json['tickets'][0]['url'] for cust_id_x in cust_list: if cust_id_x['cust_name'] in url: cust_id = cust_id_x['cust_name'] trainlog.create(ticket_log['type'], ticket_log['json_data'], created=ticket_log['created'], done=True, cust_id=cust_id) trainlog.delete(id=ticket_log['id'], cust_id='default') print ('Copying Old Ticket data: ' , ticket_log['id'], cust_id) continue else: # New Ticket and Feedback tik_data4 = None try : tik_data4 = tickets_data_json['ticket_data']['currentAccount']['subdomain'] except KeyError as err: tik_data4 = None if tik_data4 != None: cust_id = tickets_data_json["ticket_data"]['currentAccount']['subdomain'] for cust_id_x in cust_list: if cust_id_x['cust_name'] == cust_id: trainlog.create(ticket_log['type'], ticket_log['json_data'], created=ticket_log['created'], done=True, cust_id=cust_id) trainlog.delete(id=ticket_log['id'], cust_id='default') print ('Copying New Ticket/Feedback data: ' , ticket_log['id'], cust_id) continue logging.info ('copyDefaultTrainingLog : Completed : ')
def generateNewResponse(self, cust_id): logging.info('generateNewResponse : Started : ' + str(cust_id)) self.ticket_pd.loc[:, 'response_cluster'] = -1 self.ticket_pd.loc[:, 'select_response'] = np.nan X_q = [] try: X_q = self.ticket_pd['query'] except KeyError as err: logging.error("generateNewResponse : " + str(err)) return lang = getCustomerModel().getLanguage(cust_id) X_q = self.ticket_pd['query'].apply( lambda x: self.utilspace.preprocessText(x, lang=lang, ner=True)) X_q = X_q.apply(lambda x: self.utilclass.cleanData( x, lang=lang, lowercase=True, remove_stops=True, tag_remove=True)) query_clstr_itr = int(math.sqrt(len(X_q) / 2)) #int (len(X_q) / 10) print('Query Cluster Size : ', query_clstr_itr) if (query_clstr_itr <= 1): return if (query_clstr_itr > 15): query_clstr_itr = 15 self.ticket_pd['query_cluster'], _, self.ticket_pd[ 'select_tags'] = self.getKMeanClusters(cust_id, X_q, query_clstr_itr) for x in range(0, query_clstr_itr): query_sub = self.ticket_pd[self.ticket_pd.query_cluster == x] resp_clst_itr = int(math.sqrt(len(query_sub) / 2)) #int(len(query_sub) / 5) print('Response Cluster Size : ', resp_clst_itr) if resp_clst_itr <= 1: continue if resp_clst_itr > 5: resp_clst_itr = 5 X_r = query_sub['response'].apply( lambda x: self.utilspace.preprocessText(x, lang=lang, ner=True )) X_r2 = X_r.apply( lambda x: self.utilclass.cleanData(x, lang=lang, lowercase=True, remove_stops=True, tag_remove=True)) query_sub['response_cluster'], query_sub[ 'select_response'], query_sub['response_tags'], query_sub[ 'response_summary'], query_sub[ 'response_title'] = self.getKMeanClusters_resp( cust_id, X_r2, X_r, resp_clst_itr) for index, items in query_sub.iterrows(): self.ticket_pd.loc[(self.ticket_pd['id'] == items['id']), 'response_cluster'] = int( items['response_cluster']) self.ticket_pd.loc[ (self.ticket_pd['id'] == items['id']), 'select_response'] = items['select_response'] self.ticket_pd.loc[ (self.ticket_pd['id'] == items['id']), 'response_summary'] = items['response_summary'] self.ticket_pd.loc[(self.ticket_pd['id'] == items['id']), 'response_title'] = items['response_title'] self.ticket_pd.loc[(self.ticket_pd['id'] == items['id']), 'response_tags'] = items['response_tags'] print(self.ticket_pd) logging.info('generateNewResponse : Completed : ' + str(cust_id)) return