def __init__(self):
self.conversationstates = {}
# TODO: replace getting test messages with DB messages.
# insert the correct data struct messages into db first
self.sd = Sampledata()
self.messages = self.sd.getgraphmessages()
self.messagesdict = self.createmessagesdictbykey(self.messages)
self.conversationtrees = self.buildconversationtrees(self.messages)
self.RESETMESSAGE = 'chatreset'
def setUp(self):
self.sampleconversationtrees = {
1: {123: set([124, 126]), 124: set([125]), 125: set([]),
126: set([127, 128]), 127: set([]), 128: set([129]), 129: set([])},
2: {130: set([131, 132]), 131: set([133, 134]), 132: set([135]),
133: set([]), 134: set([]), 135: set([])}}
self.rbg = ResponseBuilderGraph()
# self.rbg.setconversations(self.sampleconversationtrees)
self.sd = Sampledata()
self.messages = self.sd.getgraphmessages()
class TestResponseBuilderGraph(unittest.TestCase):
def setUp(self):
self.sampleconversationtrees = {
1: {123: set([124, 126]), 124: set([125]), 125: set([]),
126: set([127, 128]), 127: set([]), 128: set([129]), 129: set([])},
2: {130: set([131, 132]), 131: set([133, 134]), 132: set([135]),
133: set([]), 134: set([]), 135: set([])}}
self.rbg = ResponseBuilderGraph()
# self.rbg.setconversations(self.sampleconversationtrees)
self.sd = Sampledata()
self.messages = self.sd.getgraphmessages()
def test_is_child(self):
self.assertTrue(self.rbg.is_child(1, 123, 124))
self.assertTrue(self.rbg.is_child(1, 124, 125))
self.assertTrue(self.rbg.is_child(2, 132, 135))
self.assertFalse(self.rbg.is_child(1, 124, 123))
self.assertFalse(self.rbg.is_child(1, 125, 123))
self.assertFalse(self.rbg.is_child(2, 999, 123))
def test_add_node(self):
self.rbg.add_node(1, 234)
self.assertTrue(type(self.rbg.conversationtrees[1][234]) == Set)
self.rbg.add_node(1, 123)
self.assertTrue(self.rbg.conversationtrees[1][123] ==
set([124, 126]))
def test_remove_node(self):
remove_result = {123: set([124, 126]), 124: set([125]), 125: set([])}
self.rbg.remove_node(1, 126)
self.assertEqual(self.rbg.conversationtrees[1], remove_result)
remove_result = {123: set([124, 126])}
self.rbg.remove_node(1, 124)
self.assertEqual(self.rbg.conversationtrees[1], remove_result)
self.rbg.remove_node(1, 999)
self.rbg.remove_node(999, 1)
self.assertEqual(self.rbg.conversationtrees[1], remove_result)
def test_remove_edge(self):
remove_edge_result = {
123: set([126]), 124: set([125]), 125: set([]),
126: set([127, 128]), 127: set([]), 128: set([129]), 129: set([])}
self.rbg.remove_edge(1, 124)
self.assertEqual(self.rbg.conversationtrees[1], remove_edge_result)
self.rbg.remove_edge(1, 'DOESNT EXIST') # result should not change
self.assertEqual(self.rbg.conversationtrees[1], remove_edge_result)
def test_add_edge(self):
add_edge_result = {128: set([129]), 129: set([]), 123: set([124, 125, 126]),
124: set([125]), 125: set([]), 126: set([128, 127]), 127: set([])}
self.rbg.add_edge(1, 123, 125)
self.assertEqual(self.rbg.conversationtrees[1], add_edge_result)
self.rbg.add_edge(1, 123, 124) # already exists, result should not change
self.assertEqual(self.rbg.conversationtrees[1], add_edge_result)
def test_buildconversationtrees(self):
convtreesresult = {
1: {123: set([124, 126]), 124: set([125]), 125: set([]),
126: set([127, 128]), 127: set([]), 128: set([129]), 129: set([])},
2: {130: set([131, 132]), 131: set([133, 134]), 132: set([135]),
133: set([]), 134: set([]), 135: set([])}}
self.assertEqual(self.rbg.buildconversationtrees(self.messages), convtreesresult)
def test_getrootnodes(self):
rootnodes = {1: 123, 2: 130}
self.assertEqual(self.rbg.getrootnodes(self.messages), rootnodes)
def test_getchildnodes(self):
self.assertEqual(self.rbg.getchildnodes(1, 123), set([124, 126]))
self.assertEqual(self.rbg.getchildnodes(1, 999), None)
self.assertEqual(self.rbg.getchildnodes(2, 132), set([135]))
self.assertEqual(self.rbg.getchildnodes(999, 1), None)
def test_getfollowupnodes(self):
sampleconvstates = self.sd.getsampleconversationstates()
self.rbg.conversationstates = sampleconvstates
self.assertEqual(self.rbg.getfollowupnodes('bob'),
{1: set([125]), 2: set([133, 134])})
self.assertEqual(self.rbg.getfollowupnodes('hank'),
{1: set([]), 2: None})
self.assertEqual(self.rbg.getfollowupnodes('ann'),
{999: None})
self.assertEqual(self.rbg.getfollowupnodes('999'),
{})
def test_getresponseformatchingmessages(self):
messageone = MessageEntity('bob', 'Hi')
messagetwo = MessageEntity('bob', 'Not great')
messagetree = MessageEntity('bob', 'Feeling tired')
messagefour = MessageEntity('bob', 'I will!')
messagefive = MessageEntity('bob', 'Good!')
messagesix = MessageEntity('bob', '')
messageseven = MessageEntity('bob', 'BLABLABLA999')
messageeight = MessageEntity('bob', '130')
messagenine = MessageEntity('bob', '135')
# first question in a conversation
self.assertEqual(self.rbg.getresponseformessages(messageone),
[{'responseText': 'Hi! :) How are you?'}])
# the followup question
self.assertEqual(self.rbg.getresponseformessages(messagetwo),
[{'responseText': 'How come?'}])
# a repeated message should not receive another reply
self.assertFalse(self.rbg.getresponseformessages(messagetwo))
# the followup question after a repeated message should
self.assertEqual(self.rbg.getresponseformessages(messagetree),
[{'responseText': 'Aww. Get some sleep!'}])
# the final message in the chain
self.assertEqual(self.rbg.getresponseformessages(messagefour),
[{'responseText': 'Good night!'}])
# the second question repeated
self.assertFalse(self.rbg.getresponseformessages(messagefive))
# testing messages that do not occur
self.assertFalse(self.rbg.getresponseformessages(messagesix))
self.assertFalse(self.rbg.getresponseformessages(messageseven))
# first message in another conversation
self.assertEqual(self.rbg.getresponseformessages(messageeight),
[{'responseText': '130'}])
self.assertEqual(self.rbg.getresponseformessages(messageone),
[{'responseText': 'Hi! :) How are you?'}])
# parent messages have not yet been answered
self.assertFalse(self.rbg.getresponseformessages(messagenine))
# tests slightly different messages that do not match letter for letter
def test_getresponseforpartialmatchingmessages(self):
messageone = MessageEntity('bob', 'Hi how are you today!')
messagetwo = MessageEntity('bob', 'Not great QQQ')
# first question in a conversation
self.assertEqual(self.rbg.getresponseformessages(messageone),
[{'responseText': 'Hi! :) How are you?'}])
# the followup question
self.assertEqual(self.rbg.getresponseformessages(messagetwo),
[{'responseText': 'How come?'}])
def test_updateconversationstate(self):
from datetime import time, tzinfo, datetime, timedelta
self.rbg.updateconversationstate('bob', 1, 1)
self.assertEqual(self.rbg.conversationstates['bob'][1]['mostrecentquestion'], 1)
self.assertIsInstance(self.rbg.conversationstates['bob'][1]['mostrecentinteraction'], datetime)
oldtimestamp = self.rbg.conversationstates['bob'][1]['mostrecentinteraction']
self.rbg.updateconversationstate('bob', 1, 999)
self.assertEqual(self.rbg.conversationstates['bob'][1]['mostrecentquestion'], 999)
self.assertNotEqual(oldtimestamp, self.rbg.conversationstates['bob'][1]['mostrecentinteraction'])
# test if conversationstate is correctly updated after incoming an message
def test_updatedconvstateaftermessage(self):
sampleconvstates = self.sd.getsampleconversationstates()
self.rbg.conversationstates = sampleconvstates
# valid example, should update conv state
message = MessageEntity('bob', 'Good!')
self.rbg.getresponseformessages(message)
self.assertEqual(self.rbg.conversationstates['bob'][1]['mostrecentquestion'], 124)
# invalid example, should not update conv state
message = MessageEntity('bob', 'I will!')
self.rbg.getresponseformessages(message)
self.assertEqual(self.rbg.conversationstates['bob'][1]['mostrecentquestion'], 124)
# invalid example, should not update conv state
message = MessageEntity('bob', '999')
self.rbg.getresponseformessages(message)
self.assertEqual(self.rbg.conversationstates['bob'][1]['mostrecentquestion'], 124)
def test_reinitialize(self):
sampleconvstates = self.sd.getsampleconversationstates()
self.rbg.conversationstates = sampleconvstates
self.rbg.reinitialize('bob')
self.assertEqual(self.rbg.conversationstates['bob'], {})
self.rbg.conversationstates = sampleconvstates
message = MessageEntity('bob', 'chatreset')
self.rbg.getresponseformessages(message)
self.assertEqual(self.rbg.conversationstates['bob'], {})
from responseBuilder import ResponseBuilder
from database.db import Db
from database.sampledata import Sampledata
db = Db()
rb = ResponseBuilder()
sampledata = Sampledata()
print 'starting...'
class SampleMessageProtocolEntity:
def __init__(self, sender, message):
self.sender = sender
self.message = unicode(message)
def getFrom(self):
return self.sender
def getBody(self):
return self.message
db.clearTestIncomingMsg()
samplemessages = sampledata.getMessages()
testpass = 0
testfail = 0
for i, message in enumerate(samplemessages):
print 'running test ', i, ' of: ', len(samplemessages)
db.insertTestIncomingMsg({'message': message['qtext'], 'sender': 'testuser'})
class ResponseBuilderGraph:
def __init__(self):
self.conversationstates = {}
# TODO: replace getting test messages with DB messages.
# insert the correct data struct messages into db first
self.sd = Sampledata()
self.messages = self.sd.getgraphmessages()
self.messagesdict = self.createmessagesdictbykey(self.messages)
self.conversationtrees = self.buildconversationtrees(self.messages)
self.RESETMESSAGE = 'chatreset'
# Create a dict representation of all the stored messages for quick
# lookup and references. messagedict =
# { msgkey: { entiremessage}, msgkey2: {entiremessage2}, .. }
def createmessagesdictbykey(self, messages):
messagedict = {}
for message in messages:
messagedict[message['key']] = message
return messagedict
def buildconversationtrees(self, messages):
conversationtrees = {}
for message in messages:
conversationtrees.setdefault(message['conv_id'], {})
conversationtrees[message['conv_id']][message['key']] = set(message['children'])
return conversationtrees
def is_child(self, conv_id, parent, child):
try:
return child in self.conversationtrees[conv_id][parent]
except Exception:
return False
def add_node(self, conv_id, node_id):
if node_id in self.conversationtrees[conv_id]:
return
else:
self.conversationtrees[conv_id][node_id] = Set([])
# will recursively delete a node and all its child components
# however, it will not delete references to its edges!
def remove_node(self, conv_id, node):
if conv_id in self.conversationtrees:
if node in self.conversationtrees[conv_id]:
if self.conversationtrees[conv_id][node]:
for childnode in self.conversationtrees[conv_id][node]:
self.remove_node(conv_id, childnode)
del self.conversationtrees[conv_id][node]
# should be called whenever a node is removed. Otherwise, edges are kept
# in place while the nodes no longer exist
def remove_edge(self, conv_id, edge):
for node in self.conversationtrees[conv_id]:
if edge in self.conversationtrees[conv_id][node]:
self.conversationtrees[conv_id][node].remove(edge)
break
def add_edge(self, conv_id, node, edge):
if edge in self.conversationtrees[conv_id][node]:
return
else:
self.conversationtrees[conv_id][node].add(edge)
def getrootnodes(self, messages):
rootnodes = {}
for message in messages:
if message['parent'] == 0:
rootnodes[message['conv_id']] = message['key']
return rootnodes
# The child nodes of the most recently asked question of a user are the
# msgs that warrant a reply. Function returns:
# conv_id : set(keys of all childnodes eligible for a reply) }
def getfollowupnodes(self, messageSender):
followupnodes = {}
convstate = self.conversationstates.setdefault(messageSender, {})
for conv_id in convstate:
childnodes = self.getchildnodes(conv_id, convstate[conv_id]['mostrecentquestion'])
if childnodes is not None:
followupnodes[conv_id] = childnodes
else:
followupnodes[conv_id] = None
return followupnodes
def getchildnodes(self, conv_id, node):
if conv_id in self.conversationtrees:
if node in self.conversationtrees[conv_id]:
return self.conversationtrees[conv_id][node]
return None
# Flattens the eligilbe msgs: the list of rootnodes and the list with sets
# of followupnodes. Returns [ msgkey, msgkey, msgkey, ...]
def geteligiblemessages(self, rootnodes, followupnodes):
eligiblemessages = []
for node in rootnodes:
eligiblemessages.append(rootnodes[node])
for childnodesets in followupnodes:
for node in followupnodes[childnodesets]:
eligiblemessages.append(node)
return eligiblemessages
# In order to find whether the incoming message warrants a response,
# we compare it only to messages that are eligible for a response.
# PROBLEMATIC: because of the way RE \b works, this has some problems when
# the sentence ends/begins with special characters. Example:
# r'\b' + 'Not great...' + r'\b' will not match 'xx Not great... xx'
# need to test impact of stripping characters before matching, but is nasty
# workaround
def getmessagematches(self, incomingmessage, eligiblemessages, messagedict):
matches = []
for msgkey in eligiblemessages:
loweredmessage = messagedict[msgkey]['qtext'].lower()
if (re.search(r'\b' + loweredmessage + r'\b', incomingmessage)
or loweredmessage == incomingmessage):
matches.append(msgkey)
return matches
def updateconversationstate(self, messageSender, conv_id, msgkey):
self.conversationstates.setdefault(messageSender, {})
self.conversationstates[messageSender].setdefault(conv_id, {})
self.conversationstates[messageSender][conv_id] = {'mostrecentinteraction': datetime.utcnow(),
'mostrecentquestion': msgkey }
def reinitialize(self, messageSender):
if messageSender in self.conversationstates:
self.conversationstates[messageSender] = {}
return
def getresponseformessages(self, message):
returnResponses = []
messageSender = message.getFrom()
try:
messagetext = message.getBody().lower()
except Exception, e:
print 'Fail getBody, will not work for Media messages:', e
return returnResponses
if messagetext == self.RESETMESSAGE:
self.reinitialize(messageSender)
return False
rootnodes = self.getrootnodes(self.messages)
followupnodes = self.getfollowupnodes(messageSender)
eligiblemessages = self.geteligiblemessages(rootnodes, followupnodes)
escapedmessage = message.replaceEscapedCharacters(messagetext)
matches = self.getmessagematches(escapedmessage, eligiblemessages, self.messagesdict)
if matches:
for match in matches:
self.updateconversationstate(messageSender, self.messagesdict[match]['conv_id'], match)
returnResponses.append({'responseText' : self.messagesdict[match]['rtext']})
return returnResponses
6. Update the conversationstate
'''
from database.sampledata import Sampledata
from database.db import Db
from datetime import time, tzinfo, datetime, timedelta
import datetime as dt
import time
import re
import logging, sys
logging.basicConfig(stream=sys.stderr, level=logging.INFO)
# logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
db = Db()
sb = Sampledata()
messages = db.getMessages()
conversations = sb.getConversations()
resetmsg = 'chatreset'
conversationTimeoutThreshold = dt.timedelta(seconds=10)
# Keeps track of the state of different conversations, so different people
# can talk to the bot at the same time without the chat intermingling a
# response.MessageProtocolEntity.getFrom() will be key.The most recent
# interaction with the bot will be tracked to figure out if the conversation
# has timed out and should be reset. Finally, it tracks how far into the
# conversation they are.
# conversationstates = {
# m.getFrom() : [
# {conv_id : x, mostrecentinteraction: timestamp, mostrecentquestion: question_nr},
# {conv_id : x, mostrecentinteraction: timestamp, mostrecentquestion: question_nr},
5. Echo the corresponding question
6. Update the conversationstate
'''
from database.sampledata import Sampledata
from database.db import Db
from datetime import time, tzinfo, datetime, timedelta
import datetime as dt
import time
import re
import logging, sys
logging.basicConfig(stream=sys.stderr, level=logging.INFO)
# logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
db = Db()
sb = Sampledata()
messages = db.getMessages()
conversations = sb.getConversations()
resetmsg = 'chatreset'
conversationTimeoutThreshold = dt.timedelta(seconds=10)
# Keeps track of the state of different conversations, so different people
# can talk to the bot at the same time without the chat intermingling a
# response.MessageProtocolEntity.getFrom() will be key.The most recent
# interaction with the bot will be tracked to figure out if the conversation
# has timed out and should be reset. Finally, it tracks how far into the
# conversation they are.
# conversationstates = {
# m.getFrom() : [
# {conv_id : x, mostrecentinteraction: timestamp, mostrecentquestion: question_nr},
# {conv_id : x, mostrecentinteraction: timestamp, mostrecentquestion: question_nr},
from responseBuilder import ResponseBuilder
from database.db import Db
from database.sampledata import Sampledata
db = Db()
rb = ResponseBuilder()
sampledata = Sampledata()
print 'starting...'
class SampleMessageProtocolEntity:
def __init__(self, sender, message):
self.sender = sender
self.message = unicode(message)
def getFrom(self):
return self.sender
def getBody(self):
return self.message
db.clearTestIncomingMsg()
samplemessages = sampledata.getMessages()
testpass = 0
testfail = 0
for i, message in enumerate(samplemessages):
print 'running test ', i, ' of: ', len(samplemessages)
