class TranslatorPreProcessorTest(unittest.TestCase):
def setUp(self):
self.client = TestClient()
def test_pre_process_word_tagger_sentence(self):
processor = WordTaggerPreProcessor()
context = self.client.create_client_context("testid")
string = processor.process(
context,
"Python is a high-level, general-purpose programming language.")
self.assertIsNotNone(string)
self.assertEqual(
"Python NNP is VBZ a DT high-level JJ general-purpose JJ programming NN language NN",
string)
def test_pre_process_word_tagger_word(self):
processor = WordTaggerPreProcessor()
context = self.client.create_client_context("testid")
string = processor.process(context, "Python")
self.assertIsNotNone(string)
self.assertEqual("Python NN", string)
def test_pre_process_word_tagger_exception(self):
processor = MockWordTaggerPreProcessor()
context = self.client.create_client_context("testid")
self.assertEqual("Python", processor.process(context, "Python"))
def test_conversation_operations_wrong_userid(self):
config = BotConversationsConfiguration()
mgr = ConversationManager(config)
convo_dir = self.get_temp_dir() + os.sep + "storage" + os.sep + "conversations"
if os.path.exists(convo_dir):
shutil.rmtree(convo_dir)
client = TestClient()
client.add_conversation_store(convo_dir)
mgr.initialise(client.storage_factory)
client_context = client.create_client_context("user1")
conversation = mgr.get_conversation(client_context)
question1 = Question.create_from_text(client_context, "Hello There")
question1.sentence(0).response = "Hi"
conversation.record_dialog(question1)
mgr.save_conversation(client_context)
client_context2 = client.create_client_context("user2")
mgr.save_conversation(client_context2)
def test_parse_response(self):
client = TestClient()
client.create_client_context("testid")
config = BrainServiceConfiguration("rest")
config._classname = "programy.testclass"
config._method = "GET"
config._host = "localhost"
config._port = 8080
config._url = "/api/v1.0/ask"
service = ProgramyRESTService(config, api=None)
self.assertEqual(
"Hello",
service._parse_response('[{"response": {"answer": "Hello"}}]'))
def test_sentences_in_db(self):
config = SQLStorageConfiguration()
engine = SQLStorageEngine(config)
engine.initialise()
store = SQLConversationStore(engine)
store.empty()
client = TestClient()
client_context = client.create_client_context("user1")
question1 = Question.create_from_text(client_context, "Hello There")
question1.sentence(0).response = "Hi"
question1.sentence(0)._positivity = 0.5
question1.sentence(0)._subjectivity = 0.6
store._write_sentences_to_db(client_context, 1, question1)
store.commit()
question2 = Question()
store._read_sentences_from_db(client_context, 1, question2)
self.assertEqual(1, len(question2.sentences))
self.assertEqual(0.5, question2.sentences[0].positivity)
self.assertEqual(0.6, question2.sentences[0].subjectivity)
self.assertEqual(["Hello", "There"], question2.sentences[0].words)
self.assertEqual("Hi", question2.sentences[0].response)
store.empty()
def test_match_context_in_db(self):
config = SQLStorageConfiguration()
engine = SQLStorageEngine(config)
engine.initialise()
store = SQLConversationStore(engine)
store.empty()
client = TestClient()
client_context = client.create_client_context("user1")
matched_context1 = MatchContext(100,
100,
sentence="Hello",
response="Hi There")
matched_context1._matched_nodes = []
matched_context1._template_node = None
store._write_match_context_to_db(client_context, 1, matched_context1)
store.commit()
matched_context2 = MatchContext(100, 100)
store._read_match_context_from_db(client_context, 1, matched_context2)
self.assertEqual(100, matched_context2.max_search_timeout)
self.assertEqual(100, matched_context2.max_search_depth)
self.assertEqual("Hello", matched_context2.sentence)
self.assertEqual("Hi There", matched_context2.response)
store.empty()
def test_save_learnf(self):
config = FileStorageConfiguration()
tmpdir = os.path.dirname(__file__) + os.sep + "learnf"
config.learnf_storage._dirs = [tmpdir]
engine = FileStorageEngine(config)
engine.initialise()
store = FileLearnfStore(engine)
test_client = TestClient()
client_context = test_client.create_client_context("test1")
learnf_path = store._get_storage_path()
learnf_fullpath = store.create_learnf_path(client_context, learnf_path)
if os.path.exists(learnf_fullpath):
os.remove(learnf_fullpath)
self.assertFalse(os.path.exists(learnf_fullpath))
template = TemplateNode()
template.append(TemplateWordNode("Hello"))
category = LearnCategory("HELLO *", "*", "*", template)
store.save_learnf(client_context, category)
self.assertTrue(os.path.exists(learnf_fullpath))
shutil.rmtree(tmpdir)
self.assertFalse(os.path.exists(tmpdir))
def setUp(self):
storage_engine = MockStorageEngine()
client = TestClient()
self.context = client.create_client_context("TESTUSER")
self.context.brain._security._account_linker = BasicAccountLinkerService(
storage_engine)
def test_trigger_triggers(self):
config = TriggerConfiguration()
config._manager = "programytest.triggers.test_rest.MockRestTriggerManager"
config._additionals["url"] = "http://localhost/api/1.0/ask"
mgr = TriggerManager.load_trigger_manager(config)
client = TestClient()
client_context = client.create_client_context("testid")
triggered = mgr.trigger("SYSTEM_STARTUP")
self.assertTrue(triggered)
triggered = mgr.trigger("SYSTEM_STARTUP", additional={"key": "value"})
self.assertTrue(triggered)
triggered = mgr.trigger("CONVERSATION_START", client_context)
self.assertTrue(triggered)
triggered = mgr.trigger("OTHER_EVENT")
self.assertTrue(triggered)
triggered = mgr.trigger("OTHER_EVENT", client_context)
self.assertTrue(triggered)
triggered = mgr.trigger("OTHER_EVENT", client_context, additional={"key": "value"})
self.assertTrue(triggered)
def test_save_learnf_with_exception(self):
config = FileStorageConfiguration()
tmpdir = os.path.dirname(__file__) + os.sep + "learnf"
config.learnf_storage._dirs = [tmpdir]
engine = FileStorageEngine(config)
engine.initialise()
store = FileLearnfStore(engine)
test_client = TestClient()
client_context = test_client.create_client_context("test1")
learnf_path = store._get_storage_path()
learnf_fullpath = store.create_learnf_path(client_context, learnf_path)
if os.path.exists(learnf_fullpath):
os.remove(learnf_fullpath)
self.assertFalse(os.path.exists(learnf_fullpath))
pattern = ET.Element('pattern')
pattern.text = "HELLO"
topic = ET.Element('topic')
topic.text = '*'
that = ET.Element('that')
that.text = '*'
template = TemplateNode()
template.append(TemplateWordNode("Hello"))
category = LearnCategory(pattern, topic, that, template)
self.assertFalse(store.save_learnf(client_context, category))
def test_ask_question_get(self):
client = TestClient()
client.add_license_keys_store()
self._client_context = client.create_client_context("testid")
self._client_context.brain._openchatbots._openchatbots[
'CHATBOT1'] = OpenChatBot(
"openchat1", "http://localhost:5959/api/rest/v2.0/ask", "GET")
mock_data = {
"response": {
"text": "Hi there from chatbot1",
},
"status": {
"code": 200,
"text": "success"
}
}
mock_response = json.dumps(mock_data)
service = OpenChatMetaBotRESTService(
BrainServiceConfiguration("openchatbot"),
api=MockRestAPI(200, mock_response))
response = service.ask_question(self._client_context, "chatbot1 Hello")
self.assertIsNotNone(response)
self.assertEquals(
'{"response": {"text": "Hi there from chatbot1"}, "status": {"code": 200, "text": "success"}}',
response)
def test_get_default_response_nothing(self):
config = ServiceConfiguration.from_data("generic", "test", "category")
service = Service(config)
client = TestClient()
client_context = client.create_client_context("testuser")
self.assertEqual("Service failed to return valid response",
service._get_default_response(client_context))
def test_load_logs(self):
config = RedisStorageConfiguration()
engine = RedisStorageEngine(config)
engine.initialise()
store = RedisLogsStore(engine)
client = TestClient()
client_context = client.create_client_context("user1")
conversation = Conversation(client_context)
error_log = {"error": "logger ERROR log"}
warning_log = {"warning": "logger WARNING log"}
info_log = {"info": "logger INFO log"}
debug_log = {"debug": "logger DEBUG log"}
conversation.append_log(error_log)
conversation.append_log(warning_log)
conversation.append_log(info_log)
conversation.append_log(debug_log)
store.store_logs(client_context, conversation)
logInfo = store.load_logs(client_context)
self.assertIsNotNone(logInfo)
logs = logInfo['logs']
self.assertIsNotNone(logs)
self.assertEqual(4, len(logs))
self.assertEqual({"error": "logger ERROR log"}, logs[0])
def assert_just_conversation_storage(self,
store,
can_empty=True,
test_load=True):
if can_empty is True:
store.empty()
client = TestClient()
client_context = client.create_client_context("user1")
conversation = Conversation(client_context)
question1 = Question.create_from_text(client_context, "Hello There")
question1.sentence(0).response = "Hi"
question1.sentence(0).matched_context = MatchContext(
max_search_depth=99,
max_search_timeout=99,
matched_nodes=[
Match(Match.WORD, PatternWordNode("Hello"), "Hello")
],
template_node=None,
sentence="HELLO",
response="Hi There")
conversation.record_dialog(question1)
store.store_conversation(client_context, conversation)
def test_get_conversation_multi_client(self):
config = BotConversationsConfiguration()
mgr = ConversationManager(config)
if os.path.exists("./storage/conversations"):
shutil.rmtree("./storage/conversations")
client = TestClient()
client.add_conversation_store("./storage/conversations")
mgr.initialise(client.storage_factory)
mgr.configuration._multi_client = True
client_context = client.create_client_context("user1")
conversation = mgr.get_conversation(client_context)
question1 = Question.create_from_text(client_context, "Hello There")
question1.sentence(0).response = "Hi"
conversation.record_dialog(question1)
mgr.save_conversation(client_context)
conversation = mgr.get_conversation(client_context)
self.assertEqual(len(mgr.conversations), 1)
if os.path.exists("./storage/conversations"):
shutil.rmtree("./storage/conversations")
def test_save_learnf_same_pattern(self):
config = RedisStorageConfiguration()
engine = RedisStorageEngine(config)
engine.initialise()
store = RedisLearnfStore(engine)
test_client = TestClient()
client_context = test_client.create_client_context("test1")
pattern = ET.Element('pattern')
pattern.text = "HELLO"
topic = ET.Element('topic')
topic.text = '*'
that = ET.Element('that')
that.text = '*'
template = TemplateNode()
template.append(TemplateWordNode("Hello"))
category = LearnCategory(pattern, topic, that, template)
is_replace = False
store.save_learnf(client_context, category, is_replace)
is_replace = True
store.save_learnf(client_context, category, is_replace)
def test_save_then_load(self):
tmpdir = FileStorageConfiguration.get_temp_dir()
fullpath = tmpdir + os.sep + "braintree/braintree.bin"
if os.path.exists(fullpath):
os.remove(fullpath)
factory = self.create_storage_factory(fullpath)
config = BrainBinariesConfiguration()
manager = BinariesManager(config)
self.assertIsNotNone(manager)
test_client = TestClient()
client_context = test_client.create_client_context("test1")
self.assertTrue(
manager.save_binary(factory, client_context.bot.brain.aiml_parser))
self.assertTrue(os.path.exists(fullpath))
aiml_parser2 = manager.load_binary(factory)
self.assertIsNotNone(aiml_parser2)
if os.path.exists(fullpath):
os.remove(fullpath)
def assert_modify_conversation_data(self, store):
client = TestClient()
client_context = client.create_client_context("user1")
conversation = Conversation(client_context)
question1 = Question.create_from_text(client_context, "Hello There")
question1.sentence(0).response = "Hi"
conversation.record_dialog(question1)
store.modify_conversation_data(client_context, conversation)
store.commit()
debugInfo, _ = store.debug_conversation_data(client_context)
self.assertIsNotNone(debugInfo)
self.assertIsNotNone(debugInfo['conversations'])
conversations = debugInfo['conversations']
self.assertIsNotNone(conversations['questions'])
self.assertEqual(1, len(conversations['questions']))
question = conversations['questions'][0]
self.assertEqual(1, len(question['sentences']))
sentence = question['sentences'][0]
self.assertEqual("Hello There", sentence['question'])
self.assertEqual("Hi", sentence['response'])
def test_context_dump(self):
client = TestClient("clientid")
client_context = client.create_client_context("testid")
self.assertIsNone(client_context.log_level)
client_context.dump()
def test_load_save_binaries(self):
yaml = YamlConfigurationFile()
self.load_os_specific_configuration(yaml, "test_secure_brain.yaml",
"test_secure_brain.windows.yaml")
brains_section = yaml.get_section("brains")
brain_section = yaml.get_section("brain", brains_section)
brain_config = BrainConfiguration()
brain_config.load_configuration(yaml, brain_section, ".")
brain_config.binaries._save_binary = True
brain_config.binaries._load_binary = False
client = TestClient()
client_context = client.create_client_context("testid")
brain1 = Brain(client_context.bot, brain_config)
self.assertIsNotNone(brain1)
brain_config.binaries._save_binary = False
brain_config.binaries._load_binary = True
brain2 = Brain(client_context.bot, brain_config)
self.assertIsNotNone(brain2)
def test_ask_question_post_with_authorization(self):
client = TestClient()
client.add_license_keys_store()
self._client_context = client.create_client_context("testid")
self._client_context.brain._openchatbots._openchatbots['CHATBOT1'] = OpenChatBot("openchat1",
"http://localhost:5959/api/rest/v2.0/ask",
["POST"],
"Basic",
"1234567890")
mock_data = {"response": {
"text": "Hi there from chatbot1",
},
"status": {"code": 200, "text": "success"}
}
mock_response = json.dumps(mock_data)
service_config = BrainServiceConfiguration("openchatbot")
service_config._method = 'POST'
service = OpenChatRESTService(service_config, api=MockRestAPI(200, mock_response))
response = service.ask_question(self._client_context, "chatbot1 Hello")
self.assertIsNotNone(response)
self.assertEqual("Hi there from chatbot1", response)
def test_read_write_question_properties_in_db(self):
config = SQLStorageConfiguration()
engine = SQLStorageEngine(config)
engine.initialise()
store = SQLConversationStore(engine)
client = TestClient()
client_context = client.create_client_context("user1")
store.empty()
properties1 = {"key1": "value1", "key2": "value2"}
store._write_properties_to_db(client_context, 1, 2,
ConversationPropertyDAO.QUESTION,
properties1)
store.commit()
properties2 = {}
store._read_properties_from_db(client_context, 1, 2,
ConversationPropertyDAO.QUESTION,
properties2)
self.assertEqual({"key1": "value1", "key2": "value2"}, properties2)
store.empty()
def test_init_context_and_converstion(self):
client = TestClient()
client_context = client.create_client_context("tesuser")
convo = Convo(client_context)
question = Question.create_from_text(client_context, "Hello world")
question.current_sentence()._response = "Hello matey"
convo.record_dialog(question)
conversation = Conversation(client_context, convo)
self.assertIsNotNone(conversation)
self.assertEqual(client_context.client.id, conversation.clientid)
self.assertEqual(client_context.userid, conversation.userid)
self.assertEqual(client_context.bot.id, conversation.botid)
self.assertEqual(client_context.brain.id, conversation.brainid)
self.assertIsNotNone(conversation.conversation)
doc = conversation.to_document()
self.assertEqual({'clientid': 'testclient', 'userid': 'tesuser', 'botid': 'bot', 'brainid': 'brain', 'conversation': {'client_context': {'clientid': 'testclient', 'userid': 'tesuser', 'botid': 'bot', 'brainid': 'brain', 'depth': 0}, 'questions': [{'sentences': [{'question': 'Hello world', 'response': 'Hello matey'}], 'srai': False, 'properties': {}, 'current_sentence_no': -1}], 'max_histories': 100, 'properties': {'topic': '*'}}},
doc)
conversation2 = Conversation.from_document(client_context, doc)
self.assertIsNotNone(conversation2)
self.assertEqual(conversation2.clientid, conversation.clientid)
self.assertEqual(conversation2.userid, conversation.userid)
self.assertEqual(conversation2.botid, conversation.botid)
self.assertEqual(conversation2.brainid, conversation.brainid)
def test_remove_conversation_invalid_userid(self):
config = BotConversationsConfiguration()
mgr = ConversationManager(config)
if os.path.exists("./storage/conversations"):
shutil.rmtree("./storage/conversations")
client = TestClient()
client.add_conversation_store("./storage/conversations")
mgr.initialise(client.storage_factory)
client_context = client.create_client_context("user1")
self.assertEqual(len(mgr.conversations), 0)
conversation = mgr.get_conversation(client_context)
question1 = Question.create_from_text(client_context, "Hello There")
question1.sentence(0).response = "Hi"
conversation.record_dialog(question1)
mgr.save_conversation(client_context)
self.assertTrue(
os.path.exists("./storage/conversations/testclient_user1.conv"))
client_context._userid = "user2"
mgr.remove_conversation(client_context)
self.assertTrue(
os.path.exists("./storage/conversations/testclient_user1.conv"))
self.assertFalse(
os.path.exists("./storage/conversations/testclient_user2.conv"))
if os.path.exists("./storage/conversations"):
shutil.rmtree("./storage/conversations")
def test_save_braintree(self):
config = FileStorageConfiguration()
config._categories_storage = FileStoreConfiguration(dirs=[os.path.dirname(__file__) + os.sep + "data" + os.sep + "categories"], extension="aiml", subdirs=False, format="xml", encoding="utf-8", delete_on_start=False)
tmpdir = os.path.dirname(__file__) + os.sep + "braintree"
tmpfile = tmpdir + os.sep + "braintree.xml"
config.braintree_storage._dirs = [tmpfile]
config.braintree_storage._has_single_file = True
engine = FileStorageEngine(config)
engine.initialise()
store = FileBraintreeStore(engine)
path = store._get_dir_from_path(store._get_storage_path())
if os.path.exists(path):
shutil.rmtree(path)
self.assertFalse(os.path.exists(path))
test_client = TestClient()
client_context = test_client.create_client_context("test1")
pattern_graph = client_context.brain.aiml_parser.pattern_parser
store.save_braintree(client_context, pattern_graph)
self.assertTrue(os.path.exists(store._get_storage_path()))
shutil.rmtree(tmpdir)
self.assertFalse(os.path.exists(tmpdir))
def assert_conversation_storage(self,
store,
can_empty=True,
test_load=True):
if can_empty is True:
store.empty()
client = TestClient()
client_context = client.create_client_context("user1")
conversation = Conversation(client_context)
question1 = Question.create_from_text(client_context, "Hello There")
question1.sentence(0).response = "Hi"
conversation.record_dialog(question1)
store.store_conversation(client_context, conversation)
store.commit()
if test_load is True:
conversation2 = Conversation(client_context)
store.load_conversation(client_context, conversation2)
self.assertEqual(1, len(conversation2.questions))
self.assertEqual(1, len(conversation2.questions[0].sentences))
self.assertEqual("Hello There",
conversation2.questions[0].sentences[0].text())
self.assertEqual("Hi",
conversation2.questions[0].sentences[0].response)
def test_store_logs(self):
self.tearDown()
config = FileStorageConfiguration()
config.logs_storage._dirs = [self._tmpdir]
engine = FileStorageEngine(config)
engine.initialise()
store = FileLogsStore(engine)
self.assertEqual(store.storage_engine, engine)
store.empty()
client = TestClient()
client_context = client.create_client_context("user1")
conversation = Conversation(client_context)
error_log = {"error": "logger ERROR log"}
warning_log = {"warning": "logger WARNING log"}
info_log = {"info": "logger INFO log"}
debug_log = {"debug": "logger DEBUG log"}
conversation.append_log(error_log)
conversation.append_log(warning_log)
conversation.append_log(info_log)
conversation.append_log(debug_log)
store.store_logs(client_context, conversation)
store.empty()
def test_brain_init_with_secure_config(self):
yaml = YamlConfigurationFile()
self.load_os_specific_configuration(yaml, "test_secure_brain.yaml", "test_secure_brain.windows.yaml")
brain_config = BrainConfiguration()
brain_config.load_configuration(yaml, os.path.dirname(__file__))
client = TestClient()
client_context = client.create_client_context("testid")
brain = Brain(client_context.bot, brain_config)
self.assertIsNotNone(brain)
self.assertIsNotNone(brain.aiml_parser)
self.assertIsNotNone(brain.denormals)
self.assertIsNotNone(brain.normals)
self.assertIsNotNone(brain.genders)
self.assertIsNotNone(brain.persons)
self.assertIsNotNone(brain.person2s)
self.assertIsNotNone(brain.properties)
self.assertIsNotNone(brain.rdf)
self.assertIsNotNone(brain.sets)
self.assertIsNotNone(brain.maps)
self.assertIsNotNone(brain.preprocessors)
self.assertIsNotNone(brain.postprocessors)
self.assertIsNotNone(brain.security)
def test_get_default_response_default_srai(self):
config = ServiceConfiguration.from_data(
"generic", "test", "category", default_srai="TEST SERVICE FAILURE")
service = Service(config)
client = TestClient()
client_context = client.create_client_context("testuser")
self.assertEqual("Default srai response",
service._get_default_response(client_context))
def setUp(self):
client = TestClient()
client.add_license_keys_store()
self.context = client.create_client_context("testid")
bot = unittest.mock.Mock()
self.context.bot = bot
self.context.brain = bot.brain
def setUp(self):
config = SQLStorageConfiguration()
storage_engine = SQLStorageEngine(config)
storage_engine.initialise()
client = TestClient()
self.context = client.create_client_context("TESTUSER")
self.context.brain._security._account_linker = BasicAccountLinkerService(
storage_engine)
