def post_process(self, output_str):
self.client = TestClient()
context = ClientContext(self.client, "testid")
context.bot = Bot(config=BotConfiguration(), client=self.client)
context.brain = context.bot.brain
context.bot.brain.denormals.add_to_lookup(" DOT COM ", [
re.compile('(^DOT COM | DOT COM | DOT COM$)', re.IGNORECASE),
'.COM '
])
context.bot.brain.denormals.add_to_lookup(
" ATSIGN ",
[re.compile('(^ATSIGN | ATSIGN | ATSIGN$)', re.IGNORECASE), '@'])
denormalize = DenormalizePostProcessor()
punctuation = FormatPunctuationProcessor()
numbers = FormatNumbersPostProcessor()
multispaces = RemoveMultiSpacePostProcessor()
emojize = EmojizePostProcessor()
output_str = denormalize.process(context, output_str)
output_str = punctuation.process(context, output_str)
output_str = numbers.process(context, output_str)
output_str = multispaces.process(context, output_str)
output_str = emojize.process(context, output_str)
return output_str
def post_process(self, output_str):
self.client = TestClient()
context = ClientContext(self.client, "testid")
config = BotConfiguration()
config.from_translator._classname = "programy.nlp.translate.textblob_translator.TextBlobTranslator"
config.from_translator._from_lang = "fr"
config.from_translator._to_lang = "en"
config.to_translator._classname = "programy.nlp.translate.textblob_translator.TextBlobTranslator"
config.to_translator._from_lang = "en"
config.to_translator._to_lang = "fr"
context.bot = Bot(config=config, client=self.client)
context.brain = context.bot.brain
context.bot.brain.denormals.add_to_lookup(" DOT COM ", [re.compile('(^DOT COM | DOT COM | DOT COM$)', re.IGNORECASE), '.COM '])
context.bot.brain.denormals.add_to_lookup(" ATSIGN ",[re.compile('(^ATSIGN | ATSIGN | ATSIGN$)', re.IGNORECASE), '@'])
denormalize = DenormalizePostProcessor()
punctuation = FormatPunctuationProcessor()
numbers = FormatNumbersPostProcessor()
multispaces = RemoveMultiSpacePostProcessor()
emojize = EmojizePostProcessor()
translate = TranslatorPostProcessor()
output_str = denormalize.process(context, output_str)
output_str = punctuation.process(context, output_str)
output_str = numbers.process(context, output_str)
output_str = multispaces.process(context, output_str)
output_str = emojize.process(context, output_str)
output_str = translate.process(context, output_str)
return output_str
def post_process(self, output_str):
self.client = TestClient()
context = ClientContext(self.client, "testid")
config = BotConfiguration()
config.from_translator._classname = "programy.translate.textblob_translator.TextBlobTranslator"
config.from_translator._from_lang = "fr"
config.from_translator._to_lang = "en"
config.to_translator._classname = "programy.translate.textblob_translator.TextBlobTranslator"
config.to_translator._from_lang = "en"
config.to_translator._to_lang = "fr"
context.bot = Bot(config=config, client=self.client)
context.brain = context.bot.brain
context.bot.brain.denormals.add_to_lookup("dot com", '.com ')
context.bot.brain.denormals.add_to_lookup("atsign", '@')
denormalize = DenormalizePostProcessor()
punctuation = FormatPunctuationProcessor()
numbers = FormatNumbersPostProcessor()
multispaces = RemoveMultiSpacePostProcessor()
emojize = EmojizePostProcessor()
translate = TranslatorPostProcessor()
output_str = denormalize.process(context, output_str)
output_str = punctuation.process(context, output_str)
output_str = numbers.process(context, output_str)
output_str = multispaces.process(context, output_str)
output_str = emojize.process(context, output_str)
output_str = translate.process(context, output_str)
return output_str
def test_format_numbers(self):
processor = FormatNumbersPostProcessor()
context = ClientContext(TestClient(), "testid")
result = processor.process(context, "23")
self.assertIsNotNone(result)
self.assertEqual("23", result)
result = processor.process(context, "23.45")
self.assertIsNotNone(result)
self.assertEqual("23.45", result)
result = processor.process(context, "23. 45")
self.assertIsNotNone(result)
self.assertEqual("23.45", result)
result = processor.process(context, "23 . 45")
self.assertIsNotNone(result)
self.assertEqual("23.45", result)
result = processor.process(context, "23,450")
self.assertIsNotNone(result)
self.assertEqual("23,450", result)
result = processor.process(context, "23, 450")
self.assertIsNotNone(result)
self.assertEqual("23,450", result)
result = processor.process(context, "23, 450, 000")
self.assertIsNotNone(result)
self.assertEqual("23,450,000", result)
def test_format_numbers(self):
processor = FormatNumbersPostProcessor()
result = processor.process(self.bot, "testid", "23")
self.assertIsNotNone(result)
self.assertEqual("23", result)
result = processor.process(self.bot, "testid", "23.45")
self.assertIsNotNone(result)
self.assertEqual("23.45", result)
result = processor.process(self.bot, "testid", "23. 45")
self.assertIsNotNone(result)
self.assertEqual("23.45", result)
result = processor.process(self.bot, "testid", "23 . 45")
self.assertIsNotNone(result)
self.assertEqual("23.45", result)
result = processor.process(self.bot, "testid", "23,450")
self.assertIsNotNone(result)
self.assertEqual("23,450", result)
result = processor.process(self.bot, "testid", "23, 450")
self.assertIsNotNone(result)
self.assertEqual("23,450", result)
result = processor.process(self.bot, "testid", "23, 450, 000")
self.assertIsNotNone(result)
self.assertEqual("23,450,000", result)
def test_format_numbers(self):
processor = FormatNumbersPostProcessor()
context = ClientContext(TestClient(), "testid")
result = processor.process(context, "23")
self.assertIsNotNone(result)
self.assertEqual("23", result)
result = processor.process(context, "23.45")
self.assertIsNotNone(result)
self.assertEqual("23.45", result)
result = processor.process(context, "23. 45")
self.assertIsNotNone(result)
self.assertEqual("23.45", result)
result = processor.process(context, "23 . 45")
self.assertIsNotNone(result)
self.assertEqual("23.45", result)
result = processor.process(context, "23,450")
self.assertIsNotNone(result)
self.assertEqual("23,450", result)
result = processor.process(context, "23, 450")
self.assertIsNotNone(result)
self.assertEqual("23,450", result)
result = processor.process(context, "23, 450, 000")
self.assertIsNotNone(result)
self.assertEqual("23,450,000", result)
def post_process(self, output_str):
context = ClientContext(TestClient(), "testid")
context.bot = Bot(config=BotConfiguration())
context.brain = context.bot.brain
context.bot.brain.denormals.process_splits([" dot com ", ".com"])
context.bot.brain.denormals.process_splits([" atsign ", "@"])
denormalize = DenormalizePostProcessor()
punctuation = FormatPunctuationProcessor()
numbers = FormatNumbersPostProcessor()
multispaces = RemoveMultiSpacePostProcessor()
output_str = denormalize.process(context, output_str)
output_str = punctuation.process(context, output_str)
output_str = numbers.process(context, output_str)
output_str = multispaces.process(context, output_str)
return output_str
def post_process(self, output_str):
context = ClientContext(TestClient(), "testid")
context.bot = Bot(config=BotConfiguration())
context.brain = context.bot.brain
context.bot.brain.denormals.process_splits([" dot com ",".com"])
context.bot.brain.denormals.process_splits([" atsign ","@"])
denormalize = DenormalizePostProcessor()
punctuation = FormatPunctuationProcessor()
numbers = FormatNumbersPostProcessor()
multispaces = RemoveMultiSpacePostProcessor()
emojize = EmojizePreProcessor()
output_str = denormalize.process(context, output_str)
output_str = punctuation.process(context, output_str)
output_str = numbers.process(context, output_str)
output_str = multispaces.process(context, output_str)
output_str = emojize.process(context, output_str)
return output_str
class PostProcessingTests(unittest.TestCase):
def setUp(self):
self.bot = Bot(Brain(BrainConfiguration()), config=BotConfiguration())
self.bot.brain.denormals.process_splits([" dot com ", ".com"])
self.bot.brain.denormals.process_splits([" atsign ", "@"])
self.denormalize = DenormalizePostProcessor()
self.punctuation = FormatPunctuationProcessor()
self.numbers = FormatNumbersPostProcessor()
self.multispaces = RemoveMultiSpacePostProcessor()
def post_process(self, output_str):
output_str = self.denormalize.process(self.bot, "testid", output_str)
output_str = self.punctuation.process(self.bot, "testid", output_str)
output_str = self.numbers.process(self.bot, "testid", output_str)
output_str = self.multispaces.process(self.bot, "testid", output_str)
return output_str
def test_post_cleanup(self):
result = self.post_process("Hello World")
self.assertIsNotNone(result)
self.assertEqual("Hello World", result)
result = self.post_process("Hello World . This is It! ")
self.assertIsNotNone(result)
self.assertEqual("Hello World. This is It!", result)
result = self.post_process("Is the result 23 . 45 ?")
self.assertIsNotNone(result)
self.assertEqual("Is the result 23.45?", result)
result = self.post_process(
"My email address is ybot atsign programy dot com")
self.assertIsNotNone(result)
self.assertEqual("My email address is [email protected]", result)
result = self.post_process("He said ' Hello World '.")
self.assertIsNotNone(result)
self.assertEqual("He said 'Hello World'.", result)
def post_process(self, output_str):
self.client = TestClient()
context = ClientContext(self.client, "testid")
context.bot = Bot(config=BotConfiguration(), client=self.client)
context.brain = context.bot.brain
context.bot.brain.denormals.add_to_lookup("dot com", '.com ')
context.bot.brain.denormals.add_to_lookup("atsign", '@')
denormalize = DenormalizePostProcessor()
punctuation = FormatPunctuationProcessor()
numbers = FormatNumbersPostProcessor()
multispaces = RemoveMultiSpacePostProcessor()
emojize = EmojizePostProcessor()
output_str = denormalize.process(context, output_str)
output_str = punctuation.process(context, output_str)
output_str = numbers.process(context, output_str)
output_str = multispaces.process(context, output_str)
output_str = emojize.process(context, output_str)
return output_str