def test_replaces_mixed_whitespace(self):
f = normalizers.WhitespaceNormalizer()
# leading
self.assertEqual(f('\n\t foo'), 'foo')
self.assertEqual(f('\n \tfoo bar'), 'foo bar')
# trailing
self.assertEqual(f('foo \n\n'), 'foo')
self.assertEqual(f('foo bar \t\n '), 'foo bar')
# leading, trailing, and internal
self.assertEqual(f('\n\t foo \n\n'), 'foo')
self.assertEqual(f('\t \nfoo \t \nbar \n\t '), 'foo bar')
def test_replaces_internal_whitespace(self):
f = normalizers.WhitespaceNormalizer()
# space character
self.assertEqual(f('foo bar'), 'foo bar')
self.assertEqual(f('foo bar'), 'foo bar')
self.assertEqual(f('foo bar'), 'foo bar')
# tab character
self.assertEqual(f('foo\tbar'), 'foo bar')
self.assertEqual(f('foo\t\tbar'), 'foo bar')
self.assertEqual(f('foo\t\t\tbar'), 'foo bar')
# newline character
self.assertEqual(f('foo\nbar'), 'foo bar')
self.assertEqual(f('foo\n\nbar'), 'foo bar')
self.assertEqual(f('foo\n\n\nbar'), 'foo bar')
def test_strips_leading_whitespace(self):
f = normalizers.WhitespaceNormalizer()
# space character
self.assertEqual(f(' foo'), 'foo')
self.assertEqual(f(' foo'), 'foo')
self.assertEqual(f(' foo'), 'foo')
self.assertEqual(f(' foo bar'), 'foo bar')
self.assertEqual(f(' foo bar'), 'foo bar')
self.assertEqual(f(' foo bar'), 'foo bar')
# tab character
self.assertEqual(f('\tfoo'), 'foo')
self.assertEqual(f('\t\tfoo'), 'foo')
self.assertEqual(f('\t\t\tfoo'), 'foo')
self.assertEqual(f('\tfoo bar'), 'foo bar')
self.assertEqual(f('\t\tfoo bar'), 'foo bar')
self.assertEqual(f('\t\t\tfoo bar'), 'foo bar')
# newline character
self.assertEqual(f('\nfoo'), 'foo')
self.assertEqual(f('\n\nfoo'), 'foo')
self.assertEqual(f('\n\n\nfoo'), 'foo')
self.assertEqual(f('\nfoo bar'), 'foo bar')
self.assertEqual(f('\n\nfoo bar'), 'foo bar')
self.assertEqual(f('\n\n\nfoo bar'), 'foo bar')
def test_strips_trailing_whitespace(self):
f = normalizers.WhitespaceNormalizer()
# space character
self.assertEqual(f('foo '), 'foo')
self.assertEqual(f('foo '), 'foo')
self.assertEqual(f('foo '), 'foo')
self.assertEqual(f('foo bar '), 'foo bar')
self.assertEqual(f('foo bar '), 'foo bar')
self.assertEqual(f('foo bar '), 'foo bar')
# tab character
self.assertEqual(f('foo\t'), 'foo')
self.assertEqual(f('foo\t\t'), 'foo')
self.assertEqual(f('foo\t\t\t'), 'foo')
self.assertEqual(f('foo bar\t'), 'foo bar')
self.assertEqual(f('foo bar\t\t'), 'foo bar')
self.assertEqual(f('foo bar\t\t\t'), 'foo bar')
# newline character
self.assertEqual(f('foo\n'), 'foo')
self.assertEqual(f('foo\n\n'), 'foo')
self.assertEqual(f('foo\n\n\n'), 'foo')
self.assertEqual(f('foo bar\n'), 'foo bar')
self.assertEqual(f('foo bar\n\n'), 'foo bar')
self.assertEqual(f('foo bar\n\n\n'), 'foo bar')
def test_leaves_normal_text_alone(self):
f = normalizers.WhitespaceNormalizer()
self.assertEqual(f('foo'), 'foo')
self.assertEqual(f('foo bar'), 'foo bar')