def test_equals_attribs(self): self._client_context.brain._sets_collection.add_set( "TEST1", { "VALUE1": [["VALUE1"]], "VALUE2": [["VALUE2"]], "VALUE3": [["VALUE3"]], "VALUE4": [["VALUE4"]] }, "teststore") node1 = PatternSetNode({"name": "test1"}, "") node2 = PatternSetNode({"name": "test1"}, "", userid="testid") node3 = PatternSetNode({"name": "test1"}, "", userid="testid2") match1 = node1.equals(self._client_context, Sentence(self._client_context, 'VALUE1'), 0) self.assertIsNotNone(match1) self.assertTrue(match1.matched) match2 = node2.equals(self._client_context, Sentence(self._client_context, 'VALUE1'), 0) self.assertIsNotNone(match2) self.assertTrue(match2.matched) match3 = node3.equals(self._client_context, Sentence(self._client_context, 'VALUE1'), 0) self.assertIsNotNone(match3) self.assertFalse(match3.matched)
def test_check_child_is_wildcard_hash(self): wildcard = MockPatternWildCardNode("*") self.assertIsNotNone(wildcard) wildcard._0ormore_hash = PatternZeroOrMoreWildCardNode('#') wildcard._0ormore_hash._template = PatternTemplateNode(TemplateNode()) context = MatchContext(max_search_depth=100, max_search_timeout=-1, tokenizer=self._client_context.brain.tokenizer) sentence = Sentence(self._client_context.brain.tokenizer, "TEST SENTENCE") match = wildcard.check_child_is_wildcard("", self._client_context, context, sentence, 0, Match.WORD, 0) self.assertIsNotNone(match) context = MatchContext(max_search_depth=100, max_search_timeout=-1, tokenizer=self._client_context.brain.tokenizer) sentence = Sentence(self._client_context.brain.tokenizer, "TEST") match = wildcard.check_child_is_wildcard("", self._client_context, context, sentence, 0, Match.WORD, 0) self.assertIsNotNone(match)
def test_sentence_creation_one_word(self): sentence = Sentence(self._bot.brain.tokenizer, "One") self.assertIsNotNone(sentence) self.assertEqual(1, sentence.num_words()) with self.assertRaises(Exception): sentence.sentence.word(1) self.assertEqual("One", sentence.text())
def test_equals_template(self): self._client_context.brain.regex_templates.add_regex( "LEGION", re.compile("^LEGION$", re.IGNORECASE)) node1 = PatternRegexNode({"template": "LEGION"}, "") node2 = PatternRegexNode({"template": "LEGION"}, "", userid="testid") node3 = PatternRegexNode({"template": "LEGION"}, "", userid="testid2") match1 = node1.equals( self._client_context, Sentence(self._client_context.brain.tokenizer, 'LEGION'), 0) self.assertIsNotNone(match1) self.assertTrue(match1.matched) match2 = node2.equals( self._client_context, Sentence(self._client_context.brain.tokenizer, 'LEGION'), 0) self.assertIsNotNone(match2) self.assertTrue(match2.matched) match3 = node3.equals( self._client_context, Sentence(self._client_context.brain.tokenizer, 'LEGION'), 0) self.assertIsNotNone(match3) self.assertFalse(match3.matched)
def test_sentence_creation_one_word(self): sentence = Sentence(self._client_context, "One") self.assertIsNotNone(sentence) self.assertEqual(1, sentence.num_words()) with self.assertRaises(Exception): sentence.sentence.word(1) self.assertEqual("One", sentence.text(self._client_context))
def split_into_sentences(self, client_context, text): if client_context.bot.sentence_splitter.is_active(): if text is not None and text.strip(): all_sentences = client_context.bot.sentence_splitter.split(text) for each_sentence in all_sentences: self._sentences.append(Sentence(client_context, each_sentence)) else: self._sentences.append(Sentence(client_context, text))
def test_question_create_from_sentence(self): sentence = Sentence(self._client_context.brain.tokenizer, "One Two Three") question = Question.create_from_sentence(sentence) self.assertIsNotNone(question) self.assertEqual(1, len(question.sentences)) self.assertEqual(sentence.text(), question.sentence(0).text()) with self.assertRaises(Exception): question.sentence(1)
def test_sentence_creation_two_words(self): sentence = Sentence(self._client_context, "One Two") self.assertIsNotNone(sentence) self.assertEqual(2, sentence.num_words()) self.assertEqual("One", sentence.word(0)) self.assertEqual("Two", sentence.word(1)) with self.assertRaises(Exception): sentence.sentence.word(2) self.assertEqual("One Two", sentence.text(self._client_context))
def test_sentence_creation_two_words_diff_split_char(self): self._client_context.brain.tokenizer.split_chars = "," sentence = Sentence(self._client_context, "One,Two") self.assertIsNotNone(sentence) self.assertEqual(2, sentence.num_words()) self.assertEqual("One", sentence.word(0)) self.assertEqual("Two", sentence.word(1)) with self.assertRaises(Exception): sentence.sentence.word(2) self.assertEqual("One,Two", sentence.text(self._client_context))
def test_sentence_creation_two_words_diff_split_char(self): tokenizer = Tokenizer(",") sentence = Sentence( tokenizer, "One,Two", ) self.assertIsNotNone(sentence) self.assertEqual(2, sentence.num_words()) self.assertEqual("One", sentence.word(0)) self.assertEqual("Two", sentence.word(1)) with self.assertRaises(Exception): sentence.sentence.word(2) self.assertEqual("One,Two", sentence.text())
def test_init(self): node = PatternPriorityWordNode("test1") self.assertIsNotNone(node) self.assertFalse(node.is_root()) self.assertTrue(node.is_priority()) self.assertFalse(node.is_wildcard()) self.assertFalse(node.is_zero_or_more()) self.assertFalse(node.is_one_or_more()) self.assertFalse(node.is_set()) self.assertFalse(node.is_bot()) self.assertFalse(node.is_template()) self.assertFalse(node.is_that()) self.assertFalse(node.is_topic()) self.assertFalse(node.is_wildcard()) self.assertIsNotNone(node.children) self.assertFalse(node.has_children()) sentence = Sentence(self._client_context, "test1 test2") self.assertTrue(node.equivalent(PatternPriorityWordNode("test1"))) result = node.equals(self._client_context, sentence, 0) self.assertTrue(result.matched) result = node.equals(self._client_context, sentence, 1) self.assertFalse(result.matched) self.assertEqual(node.to_string(), "PWORD [*] [P(0)^(0)#(0)C(0)_(0)*(0)To(0)Th(0)Te(0)] word=[test1]") self.assertEqual('<priority word="test1"></priority>\n', node.to_xml(self._client_context)) node.add_child(PatternWordNode("test2")) self.assertEqual(len(node.children), 1) self.assertEqual(node.to_string(), "PWORD [*] [P(0)^(0)#(0)C(1)_(0)*(0)To(0)Th(0)Te(0)] word=[test1]") self.assertEqual('<priority word="test1"><word word="test2"></word>\n</priority>\n', node.to_xml(self._client_context))
def test_equals_userid(self): word1 = PatternPriorityWordNode("word") word2 = PatternPriorityWordNode("word", userid="testid") word3 = PatternPriorityWordNode("word", userid="testid2") match1 = word1.equals(self._client_context, Sentence(self._client_context, 'word'), 0) self.assertIsNotNone(match1) self.assertTrue(match1.matched) match2 = word2.equals(self._client_context, Sentence(self._client_context, 'word'), 0) self.assertIsNotNone(match2) self.assertTrue(match2.matched) match3 = word3.equals(self._client_context, Sentence(self._client_context, 'word'), 0) self.assertIsNotNone(match3) self.assertFalse(match3.matched)
def test_equivalent_intent(self): nlu1 = PatternNluNode({"intent": "test"}, None) nlu2 = PatternNluNode({"intent": "test"}, None, userid="testid") nlu3 = PatternNluNode({"intent": "test"}, None, userid="testid2") conversation = self._client_context.bot.get_conversation( self._client_context) self.assertIsNotNone(conversation) question = Question.create_from_text( self._client_context, "Hello", self._client_context.bot.sentence_splitter) conversation.record_dialog(question) sentence = Sentence(self._client_context.brain.tokenizer, "YEMPTY") nlu_result = '{"intents": [{"intent": "test", "score": 0.9}], "slots": []}' conversation.current_question().set_property("__SYSTEM_NLUDATA__", nlu_result) self._client_context.match_nlu = True match1 = nlu1.equals(self._client_context, sentence, 0) self.assertIsNotNone(match1) self.assertTrue(match1.matched) match2 = nlu2.equals(self._client_context, sentence, 0) self.assertIsNotNone(match2) self.assertTrue(match2.matched) match3 = nlu3.equals(self._client_context, sentence, 0) self.assertIsNotNone(match3) self.assertFalse(match3.matched)
def test_init(self): node = PatternWordNode("test1") self.assertFalse(node.is_root()) self.assertFalse(node.is_priority()) self.assertFalse(node.is_wildcard()) self.assertFalse(node.is_zero_or_more()) self.assertFalse(node.is_one_or_more()) self.assertFalse(node.is_set()) self.assertFalse(node.is_bot()) self.assertFalse(node.is_template()) self.assertFalse(node.is_that()) self.assertFalse(node.is_topic()) self.assertFalse(node.is_wildcard()) self.assertFalse(node.is_iset()) self.assertFalse(node.is_nlu()) self.assertIsNotNone(node.children) self.assertFalse(node.has_children()) sentence = Sentence(self._client_context.brain.tokenizer, "test1 test") self.assertTrue(node.equivalent(PatternWordNode("test1"))) self.assertFalse(node.equivalent(PatternWordNode("test2"))) self.assertFalse(node.equivalent(PatternBotNode([], "test1"))) result = node.equals(self._client_context, sentence, 0) self.assertTrue(result.matched) result = node.equals(self._client_context, sentence, 1) self.assertFalse(result.matched) self.assertEqual(node.to_string(), "WORD [*] [P(0)^(0)#(0)C(0)_(0)*(0)To(0)Th(0)Te(0)] word=[test1]") node.add_child(PatternWordNode("test2")) self.assertEqual(len(node.children), 1) self.assertEqual(node.to_string(), "WORD [*] [P(0)^(0)#(0)C(1)_(0)*(0)To(0)Th(0)Te(0)] word=[test1]")
def test_equals_pattern(self): node1 = PatternRegexNode({}, "^LEGION$") node2 = PatternRegexNode({}, "^LEGION$", userid="testid") node3 = PatternRegexNode({}, "^LEGION$", userid="testid2") match1 = node1.equals(self._client_context, Sentence(self._client_context.brain.tokenizer, 'LEGION'), 0) self.assertIsNotNone(match1) self.assertTrue(match1.matched) match2 = node2.equals(self._client_context, Sentence(self._client_context.brain.tokenizer, 'LEGION'), 0) self.assertIsNotNone(match2) self.assertTrue(match2.matched) match3 = node3.equals(self._client_context, Sentence(self._client_context.brain.tokenizer, 'LEGION'), 0) self.assertIsNotNone(match3) self.assertFalse(match3.matched)
def test_check_child_is_wildcard_star(self): wildcard = MockPatternWildCardNode("*") self.assertIsNotNone(wildcard) wildcard._1ormore_star = PatternOneOrMoreWildCardNode('*') wildcard._1ormore_star._template = PatternTemplateNode(TemplateNode()) context = MatchContext(max_search_depth=100, max_search_timeout=-1) sentence = Sentence(self._client_context, "TEST SENTENCE") match = wildcard.check_child_is_wildcard("", self._client_context, context, sentence, 0, Match.WORD, 0) self.assertIsNotNone(match) context = MatchContext(max_search_depth=100, max_search_timeout=-1) sentence = Sentence(self._client_context, "TEST") match = wildcard.check_child_is_wildcard("", self._client_context, context, sentence, 0, Match.WORD, 0) self.assertIsNone(match)
def test_question_set_sentencesy(self): question = Question.create_from_text(self._client_context, "") self.assertIsNotNone(question) self.assertEqual(0, len(question.sentences)) question.sentences = [ Sentence(self._client_context, "Sentence One"), Sentence(self._client_context, "Sentence Two") ] self.assertEqual(2, len(question.sentences)) self.assertEqual("Sentence One = N/A, Sentence Two = N/A", question.debug_info(self._client_context)) self.assertEquals(-1, question.current_sentence_no) question.current_sentence_no = 1 self.assertEquals(1, question.current_sentence_no)
def test_underline(self): node = PatternOneOrMoreWildCardNode("_") self.assertIsNotNone(node) self.assertFalse(node.is_root()) self.assertFalse(node.is_priority()) self.assertFalse(node.is_zero_or_more()) self.assertTrue(node.is_one_or_more()) self.assertFalse(node.is_set()) self.assertFalse(node.is_bot()) self.assertFalse(node.is_template()) self.assertFalse(node.is_that()) self.assertFalse(node.is_topic()) self.assertTrue(node.is_wildcard()) self.assertIsNotNone(node.children) self.assertFalse(node.has_children()) self.assertEqual(node.wildcard, "_") sentence = Sentence(self._client_context.brain.tokenizer, "*") self.assertTrue(node.equivalent(PatternOneOrMoreWildCardNode("_"))) result = node.equals(self._client_context, sentence, 0) self.assertFalse(result.matched) self.assertEqual( node.to_string(), "ONEORMORE [*] [P(0)^(0)#(0)C(0)_(0)*(0)To(0)Th(0)Te(0)] wildcard=[_]" ) self.assertEqual('<oneormore wildcard="_">\n</oneormore>\n', node.to_xml(self._client_context)) self.assertFalse(node.equivalent(PatternWordNode("test")))
def test_arrow(self): node = PatternZeroOrMoreWildCardNode("^") self.assertFalse(node.is_root()) self.assertFalse(node.is_priority()) self.assertTrue(node.is_zero_or_more()) self.assertFalse(node.is_one_or_more()) self.assertFalse(node.is_set()) self.assertFalse(node.is_bot()) self.assertFalse(node.is_template()) self.assertFalse(node.is_that()) self.assertFalse(node.is_topic()) self.assertTrue(node.is_wildcard()) self.assertIsNotNone(node.children) self.assertFalse(node.has_children()) sentence = Sentence(self._client_context, "*") self.assertEqual(node.wildcard, "^") self.assertTrue(node.equivalent(PatternZeroOrMoreWildCardNode("^"))) result = node.equals(self._client_context, sentence, 0) self.assertFalse(result.matched) self.assertEqual(node.to_string(), "ZEROORMORE [*] [P(0)^(0)#(0)C(0)_(0)*(0)To(0)Th(0)Te(0)] wildcard=[^]") self.assertEqual('<zerormore wildcard="^">\n</zerormore>\n', node.to_xml(self._client_context)) self.assertFalse(node.equivalent(PatternWordNode("test")))
def test_init(self): self._client_context.brain.properties.add_property("test1", "value1") node = PatternBotNode({}, "test1") self.assertIsNotNone(node) self.assertFalse(node.is_root()) self.assertFalse(node.is_priority()) self.assertFalse(node.is_wildcard()) self.assertFalse(node.is_zero_or_more()) self.assertFalse(node.is_one_or_more()) self.assertFalse(node.is_set()) self.assertTrue(node.is_bot()) self.assertFalse(node.is_template()) self.assertFalse(node.is_that()) self.assertFalse(node.is_topic()) self.assertFalse(node.is_wildcard()) self.assertTrue(node.equivalent(PatternBotNode([], "test1"))) self.assertFalse(node.equivalent(PatternBotNode([], "test2"))) sentence = Sentence(self._client_context.brain.tokenizer, "value1 value2") result = node.equals(self._client_context, sentence, 0) self.assertTrue(result.matched) self.assertEqual(0, result.word_no) result = node.equals(self._client_context, sentence, 1) self.assertFalse(result.matched) self.assertEqual(1, result.word_no) self.assertEqual(node.to_string(), "BOT [*] [P(0)^(0)#(0)C(0)_(0)*(0)To(0)Th(0)Te(0)] property=[test1]") self.assertEqual('<bot property="test1">\n</bot>', node.to_xml(self._client_context))
def test_not_equals(self): node = PatternBotNode({}, "test1") self.assertIsNotNone(node) sentence = Sentence(self._client_context, "value1 value2") result = node.equals(self._client_context, sentence, 0) self.assertFalse(result.matched)
def create_from_text(client_context, text, split=True, srai=False): question = Question(srai) if split is True: question.split_into_sentences(client_context, text) else: question.sentences.append(Sentence(client_context, text)) question.recalculate_sentinment_score(client_context) return question
def process(self, context, word_string): YLogger.debug(context, "Removing stop words...") with_stopwords = context.brain.tokenizer.texts_to_words(word_string) non_stopwords = StopWords.remove(with_stopwords) text = context.brain.tokenizer.words_to_texts(non_stopwords) sentence = Sentence(context, text) response = context.brain.ask_question(context, sentence) return response
def process(self, context, word_string): YLogger.debug(context, "Lemmatizing sentence...") unstemmed_words = context.brain.tokenizer.texts_to_words(word_string) lemmatized_words = [Lemmatizer.lemmatize(x) for x in unstemmed_words] text = context.brain.tokenizer.words_to_texts(lemmatized_words) sentence = Sentence(context, text) response = context.brain.ask_question(context, sentence) return response
def test_check_child_is_wildcard_no_wildcard_children(self): wildcard = MockPatternWildCardNode("*") self.assertIsNotNone(wildcard) context = MatchContext(max_search_depth=100, max_search_timeout=-1) sentence = Sentence(self._client_context, "TEST SENTENCE") match = wildcard.check_child_is_wildcard("", self._client_context, context, sentence, 0, Match.WORD, 0) self.assertIsNone(match)
def test_equivalent_property(self): self._client_context.brain.properties.add_property("test1", "value1") bot1 = PatternBotNode({"property": "test1"}, None) bot2 = PatternBotNode({"property": "test1"}, None, userid="testid") bot3 = PatternBotNode({"property": "test1"}, None, userid="testid2") match1 = bot1.equals(self._client_context, Sentence(self._client_context.brain.tokenizer, 'value1'), 0) self.assertIsNotNone(match1) self.assertTrue(match1.matched) match2 = bot2.equals(self._client_context, Sentence(self._client_context.brain.tokenizer, 'value1'), 0) self.assertIsNotNone(match2) self.assertTrue(match2.matched) match3 = bot3.equals(self._client_context, Sentence(self._client_context.brain.tokenizer, 'value1'), 0) self.assertIsNotNone(match3) self.assertFalse(match3.matched)
def process(self, context, word_string): YLogger.debug(context, "Stemming sentence...") stemmer = Stemmer() unstemmed_words = context.brain.tokenizer.texts_to_words(word_string) stemmed_words = [stemmer.stem(x) for x in unstemmed_words] text = context.brain.tokenizer.words_to_texts(stemmed_words) sentence = Sentence(context, text) response = context.brain.ask_question(context, sentence) return response
def test_not_equals_template(self): self._client_context.brain.regex_templates.add_regex( "LEGION", re.compile("^LEGION$", re.IGNORECASE)) node1 = PatternRegexNode({"template": "OTHER"}, "") match = node1.equals(self._client_context, Sentence(self._client_context, 'LEGION'), 0) self.assertIsNotNone(match) self.assertFalse(match.matched)
def test_check_spelling_before_false(self): spelling_config = BotSpellingConfiguration() spelling_config._classname = "programytest.spelling.test_base.MockSpellingChecker" spelling_config._check_before = False storage_factory = None spell_checker = SpellingChecker.initiate_spellchecker( spelling_config, storage_factory) client = TestClient() client_context = client.create_client_context("user1") sentence = Sentence(client_context, "Hello word") spell_checker.check_spelling_before(client_context, sentence) self.assertEqual(sentence.text(client_context), "Hello word")
def test_to_json(self): topic = PatternOneOrMoreWildCardNode("*") word1 = PatternWordNode("Hi") word2 = PatternWordNode("There") context = MatchContext(max_search_depth=100, max_search_timeout=60, template_node=TemplateWordNode("Hello")) context.add_match(Match(Match.TOPIC, topic, None)) context.add_match(Match(Match.WORD, word1, "Hi")) context.add_match(Match(Match.WORD, word2, "There")) sentence = Sentence(self._client_context, "One Two Three", matched_context=context) json_data = sentence.to_json() self.assertIsNotNone(json_data)