class ResponseTests(TestCase):
def setUp(self):
self.response = Response("A test response.")
def test_add_signature(self):
signature = Signature("Gunther Cox")
self.response.add_signature(signature)
self.assertIn(signature, self.response.signatures)
def setUp(self):
super(ResponseIntegrationTestCase, self).setUp()
statement_object = StatementObject(text='_')
statement_model = StatementModel.objects.create(text='_')
self.object = ResponseObject(statement_object.text)
self.model = ResponseModel(statement=statement_model, response=statement_model)
self.model.save()
class ResponseIntegrationTestCase(TestCase):
"""
Test case to make sure that the Django Response model
and ChatterBot Response object have a common interface.
"""
def setUp(self):
super(ResponseIntegrationTestCase, self).setUp()
statement_object = StatementObject(text='_')
statement_model = StatementModel.objects.create(text='_')
self.object = ResponseObject(statement_object.text)
self.model = ResponseModel(statement=statement_model, response=statement_model)
self.model.save()
def test_serialize(self):
object_data = self.object.serialize()
model_data = self.model.serialize()
self.assertEqual(len(object_data), len(model_data))
self.assertIn('text', object_data)
self.assertIn('text', model_data)
self.assertEqual(object_data['text'], model_data['text'])
self.assertIn('occurrence', object_data)
self.assertIn('occurrence', model_data)
self.assertEqual(object_data['occurrence'], model_data['occurrence'])
def test_mongo_to_object(self):
self.adapter.update(
Statement('Hello',
in_response_to=[
Response('Hi', occurrence=3),
Response('Hey', occurrence=6)
]
)
)
statement_data = self.adapter.statements.find_one({'text': 'Hello'})
obj = self.adapter.mongo_to_object(statement_data)
self.assertEqual(type(obj), Statement)
self.assertEqual(len(obj.in_response_to), 2)
self.assertEqual(type(obj.in_response_to[0]), Response)
self.assertEqual(type(obj.in_response_to[1]), Response)
self.assertEqual(obj.in_response_to[0].text, 'Hi')
self.assertEqual(obj.in_response_to[0].occurrence, 3)
self.assertEqual(obj.in_response_to[1].text, 'Hey')
self.assertEqual(obj.in_response_to[1].occurrence, 6)
def test_confidence_exact_match(self):
possible_choices = [
Statement('What is your quest?',
in_response_to=[Response('What is your quest?')])
]
self.adapter.chatbot.storage.filter = MagicMock(
return_value=possible_choices)
statement = Statement('What is your quest?')
match = self.adapter.get(statement)
self.assertEqual(match.confidence, 1)
def test_get_response_statements(self):
"""
Test that we are able to get a list of only statements
that are known to be in response to another statement.
"""
statement_list = [
Statement("What... is your quest?"),
Statement("This is a phone."),
Statement("A what?",
in_response_to=[Response("This is a phone.")]),
Statement("A phone.", in_response_to=[Response("A what?")])
]
for statement in statement_list:
self.adapter.update(statement)
responses = self.adapter.get_response_statements()
self.assertEqual(len(responses), 2)
self.assertIn("This is a phone.", responses)
self.assertIn("A what?", responses)
def test_filter_contains_result(self):
statement1 = Statement(
"Testing...",
in_response_to=[
Response("What are you doing?")
]
)
statement2 = Statement(
"Testing one, two, three.",
in_response_to=[
Response("Testing...")
]
)
self.adapter.update(statement1)
self.adapter.update(statement2)
results = self.adapter.filter(
in_response_to__contains="What are you doing?"
)
self.assertEqual(len(results), 1)
self.assertIn(statement1, results)
def test_update_does_not_modify_existing_statement(self):
statement = Statement("New statement")
self.adapter.update(statement)
self.adapter.read_only = True
statement.add_response(Response("New response"))
self.adapter.update(statement)
statement_found = self.adapter.find("New statement")
self.assertEqual(statement_found.text, statement.text)
self.assertEqual(statement_found.in_response_to, [])
def test_get_closest_statement(self):
"""
Note, the content of the in_response_to field for each of the
test statements is only required because the logic adapter will
filter out any statements that are not in response to a known statement.
"""
possible_choices = [
Statement("This is a lovely bog.",
in_response_to=[Response("This is a lovely bog.")]),
Statement("This is a beautiful swamp.",
in_response_to=[Response("This is a beautiful swamp.")]),
Statement("It smells like swamp.",
in_response_to=[Response("It smells like swamp.")])
]
self.adapter.context.storage.filter = MagicMock(
return_value=possible_choices)
statement = Statement("This is a lovely swamp.")
confidence, match = self.adapter.get(statement)
self.assertEqual("This is a lovely bog.", match)
def setUp(self):
super(LowConfidenceAdapterTestCase, self).setUp()
self.adapter = LowConfidenceAdapter()
# Add a mock storage adapter to the logic adapter
self.adapter.set_chatbot(self.chatbot)
possible_choices = [
Statement(
'Who do you love?',
in_response_to=[Response('I hear you are going on a quest?')]),
Statement('What is the meaning of life?',
in_response_to=[
Response('Yuck, black licorice jelly beans.')
]),
Statement('I am Iron Man.',
in_response_to=[Response('What... is your quest?')]),
Statement('What... is your quest?',
in_response_to=[Response('I am Iron Man.')]),
Statement(
'Yuck, black licorice jelly beans.',
in_response_to=[Response('What is the meaning of life?')]),
Statement('I hear you are going on a quest?',
in_response_to=[Response('Who do you love?')]),
]
self.adapter.chatbot.storage.filter = MagicMock(
return_value=possible_choices)
def test_get_closest_statement(self):
"""
Note, the content of the in_response_to field for each of the
test statements is only required because the logic adapter will
filter out any statements that are not in response to a known statement.
"""
possible_choices = [
Statement(
"Who do you love?",
in_response_to=[Response("I hear you are going on a quest?")]),
Statement("What is the meaning of life?",
in_response_to=[
Response("Yuck, black licorice jelly beans.")
]),
Statement("I am Iron Man.",
in_response_to=[Response("What... is your quest?")]),
Statement("What... is your quest?",
in_response_to=[Response("I am Iron Man.")]),
Statement(
"Yuck, black licorice jelly beans.",
in_response_to=[Response("What is the meaning of life?")]),
Statement("I hear you are going on a quest?",
in_response_to=[Response("Who do you love?")]),
]
self.adapter.context.storage.filter = MagicMock(
return_value=possible_choices)
statement = Statement("What is your quest?")
confidence, match = self.adapter.get(statement)
self.assertEqual("What... is your quest?", match)
def train(self, conversation):
"""
Train the chat bot based on the provided list of
statements that represents a single conversation.
"""
for conversation_count, text in enumerate(conversation):
if conversation_count % 1000 == 0:
print("training %d : " % conversation_count, text[0], "|",
text[1])
statement = self.get_or_create(text[1])
statement.add_response(Response(text[0]))
self.storage.update(statement)
def test_statement_added_to_recent_response_list(self):
"""
An input statement should be added to the recent response list.
"""
statement = Statement(text='Wow!', in_response_to=[Response(text='Ok')])
response = self.chatbot.get_response(statement)
response_statement = self.chatbot.storage.get_latest_response(
self.chatbot.default_conversation_id
)
self.assertIsNotNone(response_statement)
self.assertEqual(statement.text, response_statement.text)
self.assertEqual(statement.text, response)
def test_response_list_in_results(self):
"""
If a statement with response values is found using the
filter method, they should be returned as response objects.
"""
statement = Statement(
"The first is to help yourself, the second is to help others.",
in_response_to=[Response("Why do people have two hands?")])
self.adapter.update(statement)
found = self.adapter.filter(text=statement.text)
self.assertEqual(len(found[0].in_response_to), 1)
self.assertEqual(type(found[0].in_response_to[0]), Response)
def model_to_object(self, statement_model):
"""
Convert a Django model object into a ChatterBot Statement object.
"""
statement = Statement(statement_model.text)
for response_object in statement_model.in_response_to.all():
statement.add_response(Response(
response_object.response.text,
occurrence=response_object.occurrence
))
return statement
def test_filter_multiple_parameters(self):
response = Response("Why are you counting?")
statement1 = Statement("Testing...", in_response_to=[response])
statement2 = Statement("Testing one, two, three.",
in_response_to=[response])
self.adapter.update(statement1)
self.adapter.update(statement2)
results = self.adapter.filter(text=statement1.text,
in_response_to__contains=response.text)
self.assertEqual(len(results), 1)
self.assertIn(statement1, results)
def test_filter_contains_no_result(self):
statement1 = Statement(
"Testing...",
in_response_to=[
Response("What are you doing?")
]
)
self.adapter.update(statement1)
results = self.adapter.filter(
in_response_to__contains="How do you do?"
)
self.assertEqual(results, [])
def test_filter_multiple_parameters_no_results(self):
statement1 = Statement(
"Testing...",
in_response_to=[
Response("Why are you counting?")
]
)
statement2 = Statement(
"Testing one, two, three.",
in_response_to=[
Response("Testing...")
]
)
self.adapter.update(statement1)
self.adapter.update(statement2)
results = self.adapter.filter(
text="Test",
in_response_to__contains="Testing..."
)
self.assertEqual(len(results), 0)
def _objectify_response_list(self, response_list):
"""
Takes the list of response items and returns the
list converted to object versions of the responses.
"""
in_response_to = []
for item in response_list:
text = item[0]
occurrence = item[1]
in_response_to.append(Response(text, occurrence=occurrence))
return in_response_to
def deserialize_responses(self, response_list):
"""
Takes the list of response items and returns the
list converted to object versions of the responses.
"""
in_response_to = []
for response in response_list:
text = response["text"]
del (response["text"])
in_response_to.append(Response(text, **response))
return in_response_to
def deserialize_responses(self, response_list):
"""
Takes the list of response items and returns
the list converted to Response objects.
"""
proxy_statement = Statement('')
for response in response_list:
text = response['text']
del response['text']
proxy_statement.add_response(Response(text, **response))
return proxy_statement.in_response_to
def model_to_object(self, statement_model):
"""
Convert a Django model object into a ChatterBot Statement object.
"""
statement = Statement(statement_model.text,
extra_data=json.loads(statement_model.extra_data,
encoding='utf8'))
for response_object in statement_model.in_response_to.all():
statement.add_response(
Response(response_object.response.text,
occurrence=response_object.occurrence))
return statement
def test_update_modifies_existing_statement(self):
statement = Statement("New statement")
self.adapter.update(statement)
# Check the initial values
found_statement = self.adapter.find(statement.text)
self.assertEqual(len(found_statement.in_response_to), 0)
# Update the statement value
statement.add_response(Response("New response"))
self.adapter.update(statement)
# Check that the values have changed
found_statement = self.adapter.find(statement.text)
self.assertEqual(len(found_statement.in_response_to), 1)
def deserialize_responses(self, response_list):
"""
Takes the list of response items and returns
the list converted to Response objects.
"""
proxy_statement = Statement('')
for response in response_list:
data = response.copy()
text = data['text']
del (data['text'])
proxy_statement.add_response(Response(text, **data))
return proxy_statement.in_response_to
def train(self, data):
self.logger.info("Start ViFQAListTrainer training...")
for qna_pair in data:
answer_text = qna_pair.get('answer')
questions = qna_pair.get('questions')
for question_count, question_text in enumerate(questions):
answer_statement = self.get_or_create(answer_text)
question_statement = self.get_or_create(question_text)
answer_statement.add_response(Response(question_text))
self.storage.update(question_statement)
self.storage.update(answer_statement)
self.logger.info("Finished ViFQAListTrainer training!")
def train(self, filename):
self.logger.info('beginning training')
wb = load_workbook(filename, read_only=True)
ws = wb.active
statements = []
count = 0
for row in ws.rows:
if row[0].value is None:
continue
requests = [
str(r).strip()
for r in str(row[0].value).strip('; ').split(';')
]
responses = [
str(r).strip()
for r in str(row[1].value).strip('; ').split(';')
]
count += len(responses)
statements.extend(responses)
for request in requests:
for response in responses:
statement = self.get_or_create(response.strip())
statement.add_response(Response(text=request.strip()))
self.storage.update(statement)
self.logger.info('training complete {} statements'.format(count))
try:
self.download_speeches(statements)
except Exception as e:
self.logger.error('failed to load speeches')
print(e)
class ResponseIntegrationTestCase(TestCase):
"""
Test case to make sure that the Django Response model
and ChatterBot Response object have a common interface.
"""
def setUp(self):
super(ResponseIntegrationTestCase, self).setUp()
date_created = timezone.now()
statement_object = StatementObject(text='_', created_at=date_created)
statement_model = StatementModel.objects.create(
text='_', created_at=date_created)
self.object = ResponseObject(statement_object.text)
self.model = ResponseModel(statement=statement_model,
response=statement_model)
def test_serialize(self):
object_data = self.object.serialize()
model_data = self.model.serialize()
self.assertEqual(object_data, model_data)
def train(self, conversation):
"""
Train the chat bot based on the provided list of
statements that represents a single conversation.
"""
previous_statement_text = None
for conversation_count, text in enumerate(conversation):
#print_progress_bar("List Trainer", conversation_count + 1, len(conversation))
#print(1, end='')
statement = self.get_or_create(text)
if previous_statement_text:
statement.add_response(
Response(previous_statement_text)
)
previous_statement_text = statement.text
self.storage.update(statement)
def train(self):
conv_count = 0
for conv in db.conversations.find():
line_ids = conv['lines']
history = []
conv_count += 1
for line_id in line_ids:
line = db.lines.find_one({'ext_id': line_id})
statement = self.get_or_create(line['text'], line_id)
if history:
statement.add_response(Response(history[-1].text))
history.append(statement)
self.storage.update(statement)
if conv_count % 100 == 0:
logger.warn('{0} convs processed\r'.format(conv_count))
logger.warn('{0} convs processed\n'.format(conv_count))
def train(self, conversation, remove=False):
"""
Train the chat bot based on the provided list of
statements that represents a single conversation.
"""
previous_statement_text = None
for text in enumerate(conversation):
statement = self.get_or_create(text)
if previous_statement_text:
'''
previous_statement_text = previous sentence
statement = reponse to previous sentence
'''
if (remove):
statement.remove_response(text)
else:
statement.add_response(Response(previous_statement_text))
previous_statement_text = statement.text
self.storage.update(statement)
def train(self):
conversation = "hello"
previous_statement_text = None
results = self.data.get_all_generic_ingredient()
print(results)
for conversation_count, text in enumerate(conversation):
if self.show_training_progress:
utils.print_progress_bar('Recipe Trainer',
conversation_count + 1,
len(conversation))
statement = self.get_or_create(text)
if previous_statement_text:
statement.add_response(Response(previous_statement_text))
previous_statement_text = statement.text
self.chatbot.storage.update(statement)
def train(self, *corpus_paths):
# Allow a list of corpora to be passed instead of arguments
if len(corpus_paths) == 1:
if isinstance(corpus_paths[0], list):
corpus_paths = corpus_paths[0]
# Train the chat bot with each statement and response pair
for corpus_path in corpus_paths:
corpora = self.corpus.load_corpus(corpus_path)
corpus_files = self.corpus.list_corpus_files(corpus_path)
for corpus_count, corpus in enumerate(corpora):
for conversation_count, conversation in enumerate(corpus):
print_progress_bar(
str(os.path.basename(corpus_files[corpus_count])) +
" Training", conversation_count + 1, len(corpus))
previous_statement_line = []
statement_line = []
for line in conversation:
for text in line:
statement = self.get_or_create(text)
statement.add_tags(corpus.categories)
statement_line.append(statement.text)
if previous_statement_line != []:
for previous_statement_text in previous_statement_line:
statement.add_response(
Response(previous_statement_text))
self.storage.update(statement)
previous_statement_line = statement_line
statement_line = []
def get_random(self, number=1):
"""
Returns a random statement from the api.
To generate a random tweet, search twitter for recent tweets
containing the term 'random'. Then randomly select one tweet
from the current set of tweets. Randomly choose one word from
the selected random tweet, and make a second search request.
Return one random tweet selected from the search results.
"""
statements = []
tweets = self.api.GetSearch(term="random", count=5)
tweet = random.choice(tweets)
base_response = Response(text=tweet.text)
words = tweet.text.split()
word = self.choose_word(words)
# If a valid word is found, make a second search request
# TODO: What if a word is not found?
if word:
tweets = self.api.GetSearch(term=word, count=number)
if tweets:
for tweet in tweets:
# TODO: Handle non-ascii characters properly
cleaned_text = ''.join(
[i if ord(i) < 128 else ' ' for i in tweet.text]
)
statements.append(
Statement(cleaned_text, in_response_to=[base_response])
)
if number == 1:
return random.choice(statements)
return statements
def setUp(self):
self.response = Response("A test response.")