def chatbot(title="Botify"):
zlogger.log('main_views.home', f"NEWS_TICKER = {controller.NEWS_TICKER}")
forceServiceStart()
if request.method == 'POST':
msg_in = request.form.get('askBot')
bot_msg, pred_cat = controller.getBotResponse(msg_in)
controller.addChatMsg(controller.USER_ID, msg_in, request.user_agent,
request.remote_addr, pred_cat)
controller.addChatMsg(controller.BOT_ID, bot_msg)
ARGZ = {
'title': title,
'IS_TABBED_PANE': controller.IS_TABBED_PANE,
"nav_menu": controller.TABBED_NAV_MENU,
"APP_ICON_URL": controller.APP_ICON_URL,
'BOT_ID': controller.BOT_ID,
'USER_ID': controller.USER_ID,
'scrollToAnchor': 'bottomz',
'msgs': controller.MESSAGEZ,
'NEWS_TICKER': controller.NEWS_TICKER,
'ke_data': controller.KE_DATA,
'glb_data': controller.GLOBAL_DATA,
}
layout = 'layouts/layout_app.html'
page = 'one_pager.html'
return render_template('page_creator.html',
layout=layout,
page=page,
**ARGZ)
def initService():
controller.initBot()
controller.addChatMsg( controller.BOT_ID, 'Hi there. Ask me about Corona virus')
zlogger.log(src, f'Bot App Initialized {controller.bot_app}')
controller.initStreamz()
zlogger.log(src, 'Additional Streams Initialized')
def writeTo(content, dpath, dtype=zFILE, mode=MODE_WRITE):
res = STREAMZ.get(dtype, doFile)
zlogger.log("dataSource.writeTo", "dpath = {}".format(dpath))
res(
dpath,
mode=mode,
content=content,
)
def forceServiceStart():
if len(controller.NEWS_TICKER) == 0:
controller.initStreamz()
zlogger.log('main_views.home.restartStreamz',
f"NEWS_TICKER = {controller.NEWS_TICKER}")
if controller.bot_app is None:
controller.initBot()
controller.addChatMsg(controller.BOT_ID,
'Hi there. Ask me about Corona virus')
def validate(self, text_list, ylabelz_list):
def getPredictedYlabzIdxs(pred_x_idx):
return None if pred_x_idx is None else self.train_ylabz_idxs[
pred_x_idx]
zlogger.log('cosine.validate', "y.IN: {}".format(repr(ylabelz_list)))
predicted_list = np.array([self.predict([rec]) for rec in text_list])
predicted_list = np.array(
[getPredictedYlabzIdxs(x) for x in predicted_list])
return np.array(
predicted_list == np.array(ylabelz_list)).mean(), predicted_list
def initStreamz():
global KE_DATA, GLOBAL_DATA, NEWS_DATA, NEWS_TICKER
KE_DATA, GLOBAL_DATA = apiz.getLatestSummaryStats_PA()
NEWS_DATA , NEWS_TICKER = apiz.getRelatedNews()
force_numz = [ upackDataForNewsTicker(KE_DATA),
upackDataForNewsTicker(GLOBAL_DATA), ]
NEWS_TICKER = [ *NEWS_TICKER ]
zlogger.log('controller.initStreamz', f"NEWS={NEWS_TICKER}")
def dumpSave(self, fpath=None):
fpath = self.getModelFPath(fpath)
try:
with open(fpath, "wb") as fd:
pickle.dump(self.persist, fd)
zlogger.log(
"{}.model.dump".format(self.__class__),
"Model saved to file successfully - {}".format(fpath))
except:
zlogger.logError("{}.model.dump".format(self.__class__),
"Pickle to File - {}".format(fpath))
def splitTrainTest(clean_data, test_prop=0.2):
the_data = np.array(clean_data)
zlogger.log(
'splitTrainTest',
"Provided data size = {}\n{}".format(len(the_data), the_data[0]))
n_recs = len(the_data)
n_test = math.trunc(0.2 * n_recs)
# shuffle
np.random.shuffle(the_data)
# split
train_data, test_data = the_data[:(n_recs - n_test)], the_data[(n_recs -
n_test):]
#TODO: should we flatten and when
return list(train_data), list(test_data)
def loadDump(self, fpath=None):
fpath = self.getModelFPath(fpath)
def unpackPersist():
if self.persist is not None:
for k, v in self.persist.items():
setattr(self, k, v)
try:
with open(fpath, "rb") as fd:
self.persist = pickle.load(fd)
zlogger.log("{}.model.load".format(self.__class__),
"Model loaded from file successfully")
except:
zlogger.logError("{}.model.load".format(self.__class__),
"Pickle to File - {}".format(fpath))
unpackPersist()
zlogger.log("{}.model.load".format(self.__class__),
"Persist unpacked successfully")
def mapcases(title='Covid Bot'):
forceServiceStart()
ARGZ = {
'title': title,
'IS_TABBED_PANE': controller.IS_TABBED_PANE,
"nav_menu": controller.TABBED_NAV_MENU,
"APP_ICON_URL": controller.APP_ICON_URL,
'NEWS_TICKER': controller.NEWS_TICKER,
'ke_data': controller.KE_DATA,
'glb_data': controller.GLOBAL_DATA,
}
zlogger.log('route.mapcases', f"KE_DATA = {repr(controller.KE_DATA)}")
zlogger.log('route.mapcases',
f"GLOBAL_DATA = {repr(controller.GLOBAL_DATA)}")
layout = 'layouts/layout_app.html'
page = 'widget_jhu_map.html'
return render_template('page_creator.html',
layout=layout,
page=page,
**ARGZ)
def dumpLoad(self, data_path=None, data_type=None):
self.initz()
##TODO: reconcile
self.data_path = data_path
dpath = self.data_path if data_path is None else data_path
dtype = self.data_type if data_type is None else data_type
filez = self.getDumpLoadItems()
for ext, db in filez.items():
tf = "{}.{}".format(dpath, ext)
if os.path.exists(tf):
setattr(self, db, zdata_source.readFrom(tf, dtype=dtype))
zlogger.log(
'zdataset.dumpLoad',
"Loaded {} of size {}".format(tf, len(getattr(self, db))))
else:
zlogger.log('zdataset.dumpLoad', "Not Found: {}".format(tf))
self.data = self.clean_data
self.updateXIndex()
self.updateYIndex()
def predict(self, input_text):
zlogger.log('cosine.predict', "IN: {}".format(repr(input_text)))
zlogger.log('cosine.predict',
"IN.PREPROC: {}".format(repr(self.preprocessor)))
clean_input_text = self.preprocessText(input_text)
zlogger.log(
'cosine.predict', "IN.CLEAN: {} {}".format(len(clean_input_text),
repr(clean_input_text)))
# if not isinstance(clean_input_text, str):
# clean_input_text = " ".join( list( clean_input_text) )
idx = None
resp = None
if len(clean_input_text) == 0:
## assuming is a define question['corona']
clean_input_text = ['corona']
input_vec = self.model.transform(clean_input_text)
valz = cosine_similarity(input_vec, self.trained_matrix)
idx = valz.argsort()[0][-1]
zlogger.log('cosine.predict', "ANS: {}".format(idx))
# flatz = valz.flatten()
# flatz.sort()
# resp = flatz[-1]
resp = valz[0][idx]
if resp <= self._predict_threshold: ## TODO threshold it at .5
idx = None
zlogger.log('cosine.predict', "idx = {}, resp= {}".format(idx, resp))
return idx
from envbin import zlogger
import threading, atexit
def initService():
controller.initBot()
controller.addChatMsg( controller.BOT_ID, 'Hi there. Ask me about Corona virus')
zlogger.log(src, f'Bot App Initialized {controller.bot_app}')
controller.initStreamz()
zlogger.log(src, 'Additional Streams Initialized')
if __name__ == "__main__":
src = 'run.main'
zlogger.log(src, 'STARTING')
service_thread = threading.Thread( target=initService )
service_thread.start()
if app.env == 'production':
app.run()
else:
app.run(debug=True)
zlogger.log(src, 'Flask App is started')
zlogger.log(src, f"service_thread.isAlive = {service_thread.isAlive() }")
zlogger.log(src, 'FINISHED')
if isinstance(pred_cat, list):
pred_cat = pred_cat[0]
pred_cat, response = self.dset.getPredictedAtIndex(pred_cat)
# zlogger.log("bot.Predicted", "Class = {}".format( repr(pred_cat ) ) )
if isinstance(response, list):
response, _, response_src, *_ = response
rcode = self.RCODE_LEARNT_RESPONSE
if response is None:
response = random.choice(self.DONT_KNOW_RESPONSES)
return response, rcode, pred_cat
if __name__ == "__main__":
zlogger.log("botLogic.main", "Starting")
bot = ZBotLogicFlow()
bot.loadModel(90, "TFIDF_ChatBot.zmd")
while (1):
user_input = input(colored("Talk to me: ", "yellow"))
prompt = colored(">>>: ", "green")
response, rcode = bot.getResponse(user_input)
print("{} {}\n".format(
prompt, "I don't understand. Try that again"
if response is None else response))
if (rcode == -99):
Return:
'''
def writeTo(content, dpath, dtype=zFILE, mode=MODE_WRITE):
res = STREAMZ.get(dtype, doFile)
zlogger.log("dataSource.writeTo", "dpath = {}".format(dpath))
res(
dpath,
mode=mode,
content=content,
)
if __name__ == "__main__":
zlogger.log("dataSource.main", "Starting")
arange = 'FAQ responses!A1:G1000'
gsheet_id = '1EuvcPe9WXSQTsmSqhq0LWJG4xz2ZRQ1FEdnQ_LQ-_Ks' #covid_19_faq
# gsheet_id = 'covid_19_faq'
etype = ['Text File', 'PDF', 'Article', 'Site', 'Serialized', 'GSheet']
etype_i = [zFILE, zPDF, zARTICLE, zNESTED_ARTICLES, zSERIALIZED, zGSHEET]
epath = [
'example.txt', 'example.pdf',
'https://www.nation.co.ke/counties/nairobi/Police-kill-ATM-heist-mastermind/1954174-5503356-aodphx/index.html',
'https://www.standardmedia.co.ke/corporate/news', 'example.byt',
(gsheet_id, arange)
]
econtent = [
'The quick brown fox jumper over the lazy dogs.' * 7, None, None, None,
bot_app = ZBotLogicFlow()
bot_app.loadFaqDbz( faq_path, faq_typ )
bot_app.loadModel( zbot_logic.MODEL_COSINE_TFIDF, model_fpath )
def initStreamz():
global KE_DATA, GLOBAL_DATA, NEWS_DATA, NEWS_TICKER
KE_DATA, GLOBAL_DATA = apiz.getLatestSummaryStats_PA()
NEWS_DATA , NEWS_TICKER = apiz.getRelatedNews()
force_numz = [ upackDataForNewsTicker(KE_DATA),
upackDataForNewsTicker(GLOBAL_DATA), ]
NEWS_TICKER = [ *NEWS_TICKER ]
zlogger.log('controller.initStreamz', f"NEWS={NEWS_TICKER}")
if __name__ == "__main__":
src = 'ncov.main'
zlogger.log(src, "STARTING")
initBot()
initStreamz()
zlogger.log(src, "FINISHED")
return {**tmp, **tmp2}
def getPredictedAtIndex(self, y_index):
if y_index is None:
return None, None
# zlogger.log( 'zdataset.get_y-at', "IN: {}".format(y_index ) )
class_cat = self.y_labelz[y_index]
# zlogger.log( 'zdataset.get_y-at', "CAT: {}".format( class_cat ) )
return class_cat, self.faq_db.get(class_cat, None)
###########################################################
if __name__ == "__main__":
src = "dataset.main"
zlogger.log(src, ">>>>> STARTING\n")
st = "The quick brown fox jumped over the lazy dogs. This is an account of a lost dog. His name was Jazzy and he had 7 bones. Hey there! Okay, bye."
# st = nltk.sent_tokenize( st )
ds = ["The quick brown fox", "He had 7 bones"]
ps = "The brown bones" # predict text
tokz = lemmatizeTokens(st)
print("Tokens len: {}\n{}\n".format(len(tokz), tokz))
dset = ZDataset()
dset.initFromSeq(ds)
dset.preprocess()
print("Tokens len: {}\n{}\n".format(len(dset.clean_data), dset.clean_data))