def get_menu():
global MENU
if MENU:
return MENU
MENU = load_config("menu")
return MENU
def raw_callback(parsed, raw, reply_object):
print "[SR] Smartreply Callback!"
with Store(STORE) as s:
if not s['enabled']:
return
body = "null"
for part in parsed.parts:
if str(part) == "(text/plain)":
body = part.body.encode('ascii', 'ignore')
print "[SR] Parsed body to:", body
if body == "null":
return
body = EmailReplyParser.parse_reply(body)
email_message = email.message_from_string(raw)
print "[SR] To", email_message['To']
sent_to = None
for e in s['email']:
if e in email_message['To']:
sent_to = e
if sent_to is None:
return
p = model.predict_proba([body])[0]
confidence = np.max(p)
class_ = model.classes_[np.argmax(p)]
templates = load_config('templates')['templates']
print "[SR] Confidence %.2f, Class: %s" % (confidence, class_)
if confidence > float(s['threshold']) and class_ in templates:
if s['mock']:
reply = "<b>(Team Only, Confidence: %.2f)</b><br><br>" % confidence
reply += templates[class_]
reply_email(reply_object, reply, reply_one=sent_to)
else:
reply = templates[class_]
reply_email(reply_object, reply)
return
def smart_reply_panel():
prediction = None
if request.method == 'POST':
if 'text' in request.form:
text = request.form['text']
p = model.predict_proba([text])[0]
confidence = np.max(p)
class_ = model.classes_[np.argmax(p)]
prediction = (confidence, class_)
return render_template("smart_reply.html",
c=load_config(STORE),
menu=get_menu(),
prediction=prediction)
def train_model():
"""Messy code to build model.
"""
data = load_config('smartreply_data')
# data = [d for d in data if d[0] != 'whitelist']
X = [d[1] for d in data]
Y = [d[0] for d in data]
text_clf_svm = Pipeline([('vect', CountVectorizer(stop_words='english')),
('tfidf', TfidfTransformer()),
('clf-svm',
SGDClassifier(loss='log',
penalty='l2',
alpha=1e-3,
n_iter=5,
random_state=42))])
text_clf_svm = text_clf_svm.fit(X, Y)
joblib.dump(text_clf_svm, MODEL_LOC)
def templates_panel():
config = load_config(STORE)
return render_template("templates.html", menu=get_menu(), c=config)
def collect_data():
"""Messy code to download training data.
"""
c = load_config('templates')
templates = c['templates']
training_data = []
mail = imaplib2.IMAP4_SSL(IMAP_SERVER)
mail.login(MAIL_USER, MAIL_PASSWORD)
mail.select("[Gmail]/All Mail", readonly=True)
result, data = mail.search(None, '(BODY "%s")' % ("@faqbot"))
ids = data[0]
id_list = ids.split()
for idx, r_id in enumerate(id_list):
_, data = mail.fetch(r_id, "(RFC822)")
print "%i / %i (%i%%)" % (idx, len(id_list),
int(float(idx) / len(id_list) * 100))
raw_email = "null"
for d in data:
if type(d) is tuple:
if "RFC822" in d[0]:
raw_email = d[1]
flanker_msg = mime.from_string(raw_email)
body = "null"
try:
for part in flanker_msg.parts:
if str(part) == "(text/plain)":
pp = part.body.encode('ascii', 'ignore')
body = pp
except Exception as _:
pass
if body == "null":
continue
parsed_body = EmailReplyParser.read(body)
if len(parsed_body.fragments) >= 2:
if parsed_body.fragments[0].content.split()[0] == "@faqbot":
fb = parsed_body.fragments[0].content.split()[1]
original = parsed_body.fragments[1].content
lines = []
for l in original.split('\n'):
if l.startswith('> '):
tl = l.replace('>', '').strip()
if tl != '' and not (tl.startswith('On')):
lines.append(l.replace('>', ''))
key = fb
original = '\n'.join(lines)
# Now that we have this, let's make sure it's
# valid and stuff and then save it.
if key in templates:
training_data.append((key, original))
save_config(training_data, 'smartreply_data')
def whitelist_panel():
config = load_config(STORE)
return render_template("whitelist.html", menu=get_menu(), c=config)
def quill_panel():
config = load_config(STORE)
return render_template("quill.html", menu=get_menu(), c=config)