def post_process(self, output_str):
self.client = TestClient()
context = ClientContext(self.client, "testid")
context.bot = Bot(config=BotConfiguration(), client=self.client)
context.brain = context.bot.brain
context.bot.brain.denormals.add_to_lookup(" DOT COM ", [
re.compile('(^DOT COM | DOT COM | DOT COM$)', re.IGNORECASE),
'.COM '
])
context.bot.brain.denormals.add_to_lookup(
" ATSIGN ",
[re.compile('(^ATSIGN | ATSIGN | ATSIGN$)', re.IGNORECASE), '@'])
denormalize = DenormalizePostProcessor()
punctuation = FormatPunctuationProcessor()
numbers = FormatNumbersPostProcessor()
multispaces = RemoveMultiSpacePostProcessor()
emojize = EmojizePostProcessor()
output_str = denormalize.process(context, output_str)
output_str = punctuation.process(context, output_str)
output_str = numbers.process(context, output_str)
output_str = multispaces.process(context, output_str)
output_str = emojize.process(context, output_str)
return output_str
def post_process(self, output_str):
self.client = TestClient()
context = ClientContext(self.client, "testid")
config = BotConfiguration()
config.from_translator._classname = "programy.nlp.translate.textblob_translator.TextBlobTranslator"
config.from_translator._from_lang = "fr"
config.from_translator._to_lang = "en"
config.to_translator._classname = "programy.nlp.translate.textblob_translator.TextBlobTranslator"
config.to_translator._from_lang = "en"
config.to_translator._to_lang = "fr"
context.bot = Bot(config=config, client=self.client)
context.brain = context.bot.brain
context.bot.brain.denormals.add_to_lookup(" DOT COM ", [re.compile('(^DOT COM | DOT COM | DOT COM$)', re.IGNORECASE), '.COM '])
context.bot.brain.denormals.add_to_lookup(" ATSIGN ",[re.compile('(^ATSIGN | ATSIGN | ATSIGN$)', re.IGNORECASE), '@'])
denormalize = DenormalizePostProcessor()
punctuation = FormatPunctuationProcessor()
numbers = FormatNumbersPostProcessor()
multispaces = RemoveMultiSpacePostProcessor()
emojize = EmojizePostProcessor()
translate = TranslatorPostProcessor()
output_str = denormalize.process(context, output_str)
output_str = punctuation.process(context, output_str)
output_str = numbers.process(context, output_str)
output_str = multispaces.process(context, output_str)
output_str = emojize.process(context, output_str)
output_str = translate.process(context, output_str)
return output_str
def post_process(self, output_str):
self.client = TestClient()
context = ClientContext(self.client, "testid")
config = BotConfiguration()
config.from_translator._classname = "programy.translate.textblob_translator.TextBlobTranslator"
config.from_translator._from_lang = "fr"
config.from_translator._to_lang = "en"
config.to_translator._classname = "programy.translate.textblob_translator.TextBlobTranslator"
config.to_translator._from_lang = "en"
config.to_translator._to_lang = "fr"
context.bot = Bot(config=config, client=self.client)
context.brain = context.bot.brain
context.bot.brain.denormals.add_to_lookup("dot com", '.com ')
context.bot.brain.denormals.add_to_lookup("atsign", '@')
denormalize = DenormalizePostProcessor()
punctuation = FormatPunctuationProcessor()
numbers = FormatNumbersPostProcessor()
multispaces = RemoveMultiSpacePostProcessor()
emojize = EmojizePostProcessor()
translate = TranslatorPostProcessor()
output_str = denormalize.process(context, output_str)
output_str = punctuation.process(context, output_str)
output_str = numbers.process(context, output_str)
output_str = multispaces.process(context, output_str)
output_str = emojize.process(context, output_str)
output_str = translate.process(context, output_str)
return output_str
def post_process(self, output_str):
context = ClientContext(TestClient(), "testid")
context.bot = Bot(config=BotConfiguration())
context.brain = context.bot.brain
context.bot.brain.denormals.process_splits([" dot com ", ".com"])
context.bot.brain.denormals.process_splits([" atsign ", "@"])
denormalize = DenormalizePostProcessor()
punctuation = FormatPunctuationProcessor()
numbers = FormatNumbersPostProcessor()
multispaces = RemoveMultiSpacePostProcessor()
output_str = denormalize.process(context, output_str)
output_str = punctuation.process(context, output_str)
output_str = numbers.process(context, output_str)
output_str = multispaces.process(context, output_str)
return output_str
def post_process(self, output_str):
context = ClientContext(TestClient(), "testid")
context.bot = Bot(config=BotConfiguration())
context.brain = context.bot.brain
context.bot.brain.denormals.process_splits([" dot com ",".com"])
context.bot.brain.denormals.process_splits([" atsign ","@"])
denormalize = DenormalizePostProcessor()
punctuation = FormatPunctuationProcessor()
numbers = FormatNumbersPostProcessor()
multispaces = RemoveMultiSpacePostProcessor()
emojize = EmojizePreProcessor()
output_str = denormalize.process(context, output_str)
output_str = punctuation.process(context, output_str)
output_str = numbers.process(context, output_str)
output_str = multispaces.process(context, output_str)
output_str = emojize.process(context, output_str)
return output_str
class PostProcessingTests(unittest.TestCase):
def setUp(self):
self.bot = Bot(Brain(BrainConfiguration()), config=BotConfiguration())
self.bot.brain.denormals.process_splits([" dot com ", ".com"])
self.bot.brain.denormals.process_splits([" atsign ", "@"])
self.denormalize = DenormalizePostProcessor()
self.punctuation = FormatPunctuationProcessor()
self.numbers = FormatNumbersPostProcessor()
self.multispaces = RemoveMultiSpacePostProcessor()
def post_process(self, output_str):
output_str = self.denormalize.process(self.bot, "testid", output_str)
output_str = self.punctuation.process(self.bot, "testid", output_str)
output_str = self.numbers.process(self.bot, "testid", output_str)
output_str = self.multispaces.process(self.bot, "testid", output_str)
return output_str
def test_post_cleanup(self):
result = self.post_process("Hello World")
self.assertIsNotNone(result)
self.assertEqual("Hello World", result)
result = self.post_process("Hello World . This is It! ")
self.assertIsNotNone(result)
self.assertEqual("Hello World. This is It!", result)
result = self.post_process("Is the result 23 . 45 ?")
self.assertIsNotNone(result)
self.assertEqual("Is the result 23.45?", result)
result = self.post_process(
"My email address is ybot atsign programy dot com")
self.assertIsNotNone(result)
self.assertEqual("My email address is [email protected]", result)
result = self.post_process("He said ' Hello World '.")
self.assertIsNotNone(result)
self.assertEqual("He said 'Hello World'.", result)
def post_process(self, output_str):
self.client = TestClient()
context = ClientContext(self.client, "testid")
context.bot = Bot(config=BotConfiguration(), client=self.client)
context.brain = context.bot.brain
context.bot.brain.denormals.add_to_lookup("dot com", '.com ')
context.bot.brain.denormals.add_to_lookup("atsign", '@')
denormalize = DenormalizePostProcessor()
punctuation = FormatPunctuationProcessor()
numbers = FormatNumbersPostProcessor()
multispaces = RemoveMultiSpacePostProcessor()
emojize = EmojizePostProcessor()
output_str = denormalize.process(context, output_str)
output_str = punctuation.process(context, output_str)
output_str = numbers.process(context, output_str)
output_str = multispaces.process(context, output_str)
output_str = emojize.process(context, output_str)
return output_str
def test_format_punctuation(self):
processor = FormatPunctuationProcessor()
context = ClientContext(TestClient(), "testid")
result = processor.process(context, 'Hello " World "')
self.assertIsNotNone(result)
self.assertEqual('Hello "World"', result)
result = processor.process(context, '"Hello World"')
self.assertIsNotNone(result)
self.assertEqual('"Hello World"', result)
result = processor.process(context, "' Hello World '")
self.assertIsNotNone(result)
self.assertEqual("'Hello World'", result)
result = processor.process(context, '" Hello World "')
self.assertIsNotNone(result)
self.assertEqual('"Hello World"', result)
result = processor.process(context, '"This" and "That"')
self.assertIsNotNone(result)
self.assertEqual('"This" and "That"', result)
result = processor.process(context, "Hello World .")
self.assertIsNotNone(result)
self.assertEqual("Hello World.", result)
result = processor.process(context, "Hello World ,")
self.assertIsNotNone(result)
self.assertEqual("Hello World,", result)
result = processor.process(context, "Hello World :")
self.assertIsNotNone(result)
self.assertEqual("Hello World:", result)
result = processor.process(context, "Hello World ;")
self.assertIsNotNone(result)
self.assertEqual("Hello World;", result)
result = processor.process(context, "Hello World ?")
self.assertIsNotNone(result)
self.assertEqual("Hello World?", result)
result = processor.process(context, "Hello World !")
self.assertIsNotNone(result)
self.assertEqual("Hello World!", result)
result = processor.process(context, "Hello World . This is it.")
self.assertIsNotNone(result)
self.assertEqual("Hello World. This is it.", result)
result = processor.process(context, "Hello World . 23.45.")
self.assertIsNotNone(result)
self.assertEqual("Hello World. 23.45.", result)
def test_format_punctuation14(self):
processor = FormatPunctuationProcessor()
result = processor.process(self.context, "Hello World . Hi .")
self.assertIsNotNone(result)
self.assertEqual("Hello World. Hi.", result)
def test_format_punctuation5(self):
processor = FormatPunctuationProcessor()
result = processor.process(self.context, '"This" and "That"')
self.assertIsNotNone(result)
self.assertEqual('"This" and "That"', result)
def test_format_punctuation4(self):
processor = FormatPunctuationProcessor()
result = processor.process(self.context, '" Hello World "')
self.assertIsNotNone(result)
self.assertEqual('"Hello World"', result)
def test_format_punctuation_empty(self):
processor = FormatPunctuationProcessor()
result = processor.process(self.context, '')
self.assertIsNotNone(result)
self.assertEqual('', result)
def test_format_punctuation(self):
processor = FormatPunctuationProcessor()
context = ClientContext(TestClient(), "testid")
result = processor.process(context, 'Hello " World "')
self.assertIsNotNone(result)
self.assertEqual('Hello "World"', result)
result = processor.process(context, '"Hello World"')
self.assertIsNotNone(result)
self.assertEqual('"Hello World"', result)
result = processor.process(context, "' Hello World '")
self.assertIsNotNone(result)
self.assertEqual("'Hello World'", result)
result = processor.process(context, '" Hello World "')
self.assertIsNotNone(result)
self.assertEqual('"Hello World"', result)
result = processor.process(context, '"This" and "That"')
self.assertIsNotNone(result)
self.assertEqual('"This" and "That"', result)
result = processor.process(context, "Hello World .")
self.assertIsNotNone(result)
self.assertEqual("Hello World.", result)
result = processor.process(context, "Hello World ,")
self.assertIsNotNone(result)
self.assertEqual("Hello World,", result)
result = processor.process(context, "Hello World :")
self.assertIsNotNone(result)
self.assertEqual("Hello World:", result)
result = processor.process(context, "Hello World ;")
self.assertIsNotNone(result)
self.assertEqual("Hello World;", result)
result = processor.process(context, "Hello World ?")
self.assertIsNotNone(result)
self.assertEqual("Hello World?", result)
result = processor.process(context, "Hello World !")
self.assertIsNotNone(result)
self.assertEqual("Hello World!", result)
result = processor.process(context, "Hello World . This is it.")
self.assertIsNotNone(result)
self.assertEqual("Hello World. This is it.", result)
result = processor.process(context, "Hello World . 23.45.")
self.assertIsNotNone(result)
self.assertEqual("Hello World. 23.45.", result)