def process(self,
statement,
additional_response_selection_parameters=None):
"""
Takes a statement string.
Returns the equation from the statement with the mathematical terms solved.
"""
from mathparse import mathparse
input_text = statement.text
# Use the result cached by the process method if it exists
if input_text in self.cache:
cached_result = self.cache[input_text]
self.cache = {}
return cached_result
# Getting the mathematical terms within the input statement
expression = mathparse.extract_expression(
input_text, language=self.language.ISO_639.upper())
response = Statement(text=expression)
try:
response.text += ' = ' + str(
mathparse.parse(expression,
language=self.language.ISO_639.upper()))
# The confidence is 1 if the expression could be evaluated
response.confidence = 1
except mathparse.PostfixTokenEvaluationException:
response.confidence = 0
return response
def handle_matches(self, match):
"""
Returns a response statement from a matched input statement.
:param match: It is a valid matched pattern from the input statement
:type: `_sre.SRE_Match`
"""
response = Statement(text='')
from_parsed = match.group("from")
target_parsed = match.group("target")
n_statement = match.group("number")
if n_statement == 'a' or n_statement == 'an':
n_statement = '1.0'
n = mathparse.parse(n_statement, self.language.ISO_639.upper())
from_parsed, target_parsed = self.get_valid_units(
from_parsed, target_parsed)
if from_parsed is None or target_parsed is None:
response.confidence = 0.0
else:
from_value = self.unit_registry.Quantity(float(n), from_parsed)
target_value = from_value.to(target_parsed)
response.confidence = 1.0
response.text = str(target_value.magnitude)
return response
def test_learn_response(self):
previous_response = Statement(text='Define Hemoglobin.')
statement = Statement(
text='Hemoglobin is an oxygen-transport metalloprotein.')
self.chatbot.learn_response(statement, previous_response)
results = list(self.chatbot.storage.filter(text=statement.text))
self.assertIsLength(results, 1)
def test_generate_export_data(self):
self.chatbot.storage.create_many([
Statement(text='Hello, how are you?'),
Statement(text='I am good.', in_response_to='Hello, how are you?')
])
data = self.trainer._generate_export_data()
self.assertEqual([['Hello, how are you?', 'I am good.']], data)
def test_get_first_response(self):
statement_list = [
Statement(text='What... is your quest?'),
Statement(text='A what?'),
Statement(text='A quest.')
]
output = response_selection.get_first_response(Statement(text='Hello'),
statement_list)
self.assertEqual(output.text, 'What... is your quest?')
def test_get_random_response(self):
statement_list = [
Statement(text='This is a phone.'),
Statement(text='A what?'),
Statement(text='A phone.')
]
output = response_selection.get_random_response(
Statement(text='Hello'), statement_list)
self.assertTrue(output)
def test_low_confidence_options_list(self):
"""
Test the case that a high confidence response is not known.
"""
self.adapter.default_responses = [Statement(text='No')]
statement = Statement(text='Is this a tomato?')
match = self.adapter.process(statement)
self.assertEqual(match.confidence, 0)
self.assertEqual(match.text, 'No')
def test_create_many_search_in_response_to(self):
self.adapter.create_many([
Statement(text='A', search_in_response_to='a'),
Statement(text='B', search_in_response_to='b')
])
results = list(self.adapter.filter())
self.assertEqual(len(results), 2)
self.assertEqual(results[0].search_in_response_to, 'a')
self.assertEqual(results[1].search_in_response_to, 'b')
def test_create_many_text(self):
self.adapter.create_many([
Statement(text='A'),
Statement(text='B')
])
results = list(self.adapter.filter())
self.assertEqual(len(results), 2)
self.assertEqual(results[0].text, 'A')
self.assertEqual(results[1].text, 'B')
def test_confidence_no_match(self):
from unittest.mock import MagicMock
# Assume that the storage adapter returns a partial match
self.search_algorithm.chatbot.storage.filter = MagicMock(
return_value=[Statement(text='xxx', in_response_to='xxx')])
statement = Statement(text='yyy')
results = list(self.search_algorithm.search(statement))
self.assertIsLength(results, 0)
def test_create_many_tags(self):
self.adapter.create_many([
Statement(text='A', tags=['first', 'letter']),
Statement(text='B', tags=['second', 'letter'])
])
results = list(self.adapter.filter())
self.assertEqual(len(results), 2)
self.assertIn('letter', results[0].get_tags())
self.assertIn('letter', results[1].get_tags())
self.assertIn('first', results[0].get_tags())
self.assertIn('second', results[1].get_tags())
def test_order_by_text(self):
statement_a = Statement(text='A is the first letter of the alphabet.')
statement_b = Statement(text='B is the second letter of the alphabet.')
self.adapter.update(statement_b)
self.adapter.update(statement_a)
results = list(self.adapter.filter(order_by=['text']))
self.assertEqual(len(results), 2)
self.assertEqual(statement_a.text, results[0].text)
self.assertEqual(statement_b.text, results[1].text)
def test_true(self):
# Test sentences with different stopwords.
statement = Statement(text='What is matter?')
other_statement = Statement(text='What is the matter?')
value = self.compare(statement, other_statement)
self.assertAlmostEqual(value, 0.9, places=1)
# Test that text capitalization is ignored.
statement = Statement(text='Hi HoW ArE yOu?')
other_statement = Statement(text='hI hOw are YoU?')
value = self.compare(statement, other_statement)
self.assertAlmostEqual(value, 1, places=1)
def process(self,
statement,
additional_response_selection_parameters=None):
now = datetime.now()
time_features = self.time_question_features(statement.text.lower())
confidence = self.classifier.classify(time_features)
response = Statement(text='The current time is ' +
now.strftime('%I:%M %p'))
response.confidence = confidence
return response
def test_confidence_half_match(self):
from unittest.mock import MagicMock
# Assume that the storage adapter returns a partial match
self.chatbot.storage.filter = MagicMock(
return_value=[Statement(text='xxyy')])
statement = Statement(text='wwxx')
results = list(self.search_algorithm.search(statement))
self.assertIsLength(results, 1)
self.assertEqual(results[0].confidence, 0.5)
def test_different_punctuation(self):
self.chatbot.storage.create_many([
Statement(text='Who are you?'),
Statement(text='Are you good?'),
Statement(text='You are good')
])
statement = Statement(text='Are you good')
results = list(self.search_algorithm.search(statement))
self.assertEqual(len(results), 2, msg=[r.search_text for r in results])
# Note: the last statement in the list always has the highest confidence
self.assertEqual(results[-1].text, 'Are you good?')
def test_convert_to_ascii(self):
statement = Statement(
text=u'Klüft skräms inför på fédéral électoral große')
cleaned = preprocessors.convert_to_ascii(statement)
normal_text = 'Kluft skrams infor pa federal electoral groe'
self.assertEqual(cleaned.text, normal_text)
def test_consecutive_spaces_removed(self):
statement = Statement(
text='The quick brown fox jumps over the lazy dog.')
cleaned = preprocessors.clean_whitespace(statement)
normal_text = 'The quick brown fox jumps over the lazy dog.'
self.assertEqual(cleaned.text, normal_text)
def create(self, **kwargs):
"""
Creates a new statement matching the keyword arguments specified.
Returns the created statement.
"""
Statement = self.get_model('statement')
if 'tags' in kwargs:
kwargs['tags'] = list(set(kwargs['tags']))
if 'search_text' not in kwargs:
kwargs['search_text'] = self.tagger.get_text_index_string(
kwargs['text'])
if 'search_in_response_to' not in kwargs:
if kwargs.get('in_response_to'):
kwargs[
'search_in_response_to'] = self.tagger.get_text_index_string(
kwargs['in_response_to'])
inserted = self.statements.insert_one(kwargs)
kwargs['id'] = inserted.inserted_id
return Statement(**kwargs)
def __init__(self, chatbot, **kwargs):
super().__init__(chatbot, **kwargs)
from app.chatterbot_api.chatterbot.response_selection import get_first_response
self.search_algorithm_name = kwargs.get(
'search_algorithm_name',
IndexedTextSearch.name
)
self.search_algorithm = self.chatbot.search_algorithms[
self.search_algorithm_name
]
self.maximum_similarity_threshold = kwargs.get(
'maximum_similarity_threshold', 0.95
)
# By default, select the first available response
self.select_response = kwargs.get(
'response_selection_method',
get_first_response
)
default_responses = kwargs.get('default_response', [])
# Convert a single string into a list
if isinstance(default_responses, str):
default_responses = [
default_responses
]
self.default_responses = [
Statement(text=default) for default in default_responses
]
def test_clean_whitespace(self):
statement = Statement(
text='\tThe quick \nbrown fox \rjumps over \vthe \alazy \fdog\\.')
cleaned = preprocessors.clean_whitespace(statement)
normal_text = 'The quick brown fox jumps over \vthe \alazy \fdog\\.'
self.assertEqual(cleaned.text, normal_text)
def test_filter_selection(self):
"""
Test that repetitive responses are filtered out of the results.
"""
from app.chatterbot_api.chatterbot.conversation import Statement
from app.chatterbot_api.chatterbot.trainers import ListTrainer
self.chatbot.filters = (filters.get_recent_repeated_responses, )
self.trainer = ListTrainer(
self.chatbot,
show_training_progress=False
)
self.trainer.train([
'Hi',
'Hello',
'Hi',
'Hello',
'Hi',
'Hello',
'How are you?',
'I am good',
'Glad to hear',
'How are you?'
])
statement = Statement(text='Hello', conversation='training')
first_response = self.chatbot.get_response(statement)
second_response = self.chatbot.get_response(statement)
self.assertEqual('How are you?', first_response.text)
self.assertEqual('Hi', second_response.text)
def train(self, conversation):
"""
Train the chat bot based on the provided list of
statements that represents a single conversation.
"""
previous_statement_text = None
previous_statement_search_text = ''
statements_to_create = []
for conversation_count, text in enumerate(conversation):
if self.show_training_progress:
utils.print_progress_bar('List Trainer',
conversation_count + 1,
len(conversation))
statement_search_text = self.chatbot.storage.tagger.get_text_index_string(
text)
statement = self.get_preprocessed_statement(
Statement(text=text,
search_text=statement_search_text,
in_response_to=previous_statement_text,
search_in_response_to=previous_statement_search_text,
conversation='training'))
previous_statement_text = statement.text
previous_statement_search_text = statement_search_text
statements_to_create.append(statement)
self.chatbot.storage.create_many(statements_to_create)
def train(self, *corpus_paths):
from app.chatterbot_api.chatterbot.corpus import load_corpus, list_corpus_files
data_file_paths = []
# Get the paths to each file the bot will be trained with
for corpus_path in corpus_paths:
data_file_paths.extend(list_corpus_files(corpus_path))
for corpus, categories, file_path in load_corpus(*data_file_paths):
statements_to_create = []
# Train the chat bot with each statement and response pair
for conversation_count, conversation in enumerate(corpus):
if self.show_training_progress:
utils.print_progress_bar(
'Training ' + str(os.path.basename(file_path)),
conversation_count + 1, len(corpus))
previous_statement_text = None
previous_statement_search_text = ''
for text in conversation:
statement_search_text = self.chatbot.storage.tagger.get_text_index_string(
text)
statement = Statement(
text=text,
search_text=statement_search_text,
in_response_to=previous_statement_text,
search_in_response_to=previous_statement_search_text,
conversation='training')
statement.add_tags(*categories)
statement = self.get_preprocessed_statement(statement)
previous_statement_text = statement.text
previous_statement_search_text = statement_search_text
statements_to_create.append(statement)
if statements_to_create:
self.chatbot.storage.create_many(statements_to_create)
def test_search_additional_parameters(self):
"""
It should be possible to pass additional parameters in to use for searching.
"""
self.chatbot.storage.create_many([
Statement(text='A', conversation='test_1'),
Statement(text='A', conversation='test_2')
])
statement = Statement(text='A')
results = list(
self.search_algorithm.search(statement, conversation='test_1'))
self.assertIsLength(results, 1)
self.assertEqual(results[0].text, 'A')
self.assertEqual(results[0].conversation, 'test_1')
def test_search_no_results(self):
"""
An exception should be raised if there is no data to return.
"""
statement = Statement(text='What is your quest?')
with self.assertRaises(StopIteration):
next(self.search_algorithm.search(statement))
def test_no_known_responses(self):
"""
A match can be selected which has no known responses.
In this case a random response will be returned, but the confidence
should be zero because it is a random choice.
"""
from unittest.mock import MagicMock
self.chatbot.storage.update = MagicMock()
self.chatbot.storage.count = MagicMock(return_value=1)
self.chatbot.storage.get_random = MagicMock(return_value=Statement(
text='Random'))
match = self.adapter.process(Statement(text='Blah'))
self.assertEqual(match.confidence, 0)
self.assertEqual(match.text, 'Random')
def test_get_response_additional_response_selection_parameters(self):
self.chatbot.storage.create_many([
Statement('A', conversation='test_1'),
Statement('B', conversation='test_1', in_response_to='A'),
Statement('A', conversation='test_2'),
Statement('C', conversation='test_2', in_response_to='A'),
])
statement = Statement(text='A', conversation='test_3')
response = self.chatbot.get_response(
statement,
additional_response_selection_parameters={
'conversation': 'test_2'
})
self.assertEqual(response.text, 'C')
self.assertEqual(response.conversation, 'test_3')
def test_generate_response(self):
statement = Statement(
text=
'Many insects adopt a tripedal gait for rapid yet stable walking.')
response = self.chatbot.generate_response(statement)
self.assertEqual(response.text, statement.text)
self.assertEqual(response.confidence, 0)
def test_search_text_results_after_training(self):
"""
ChatterBot should return close matches to an input
string when filtering using the search_text parameter.
"""
self.chatbot.storage.create_many([
Statement('Example A for search.'),
Statement('Another example.'),
Statement('Example B for search.'),
Statement(text='Another statement.'),
])
text = self.chatbot.storage.tagger.get_text_index_string(
'Example A for search.')
results = list(self.chatbot.storage.filter(search_text=text))
self.assertEqual(len(results), 1)
self.assertEqual('Example A for search.', results[0].text)
