Example #1
0
def _split_text(text):
    '''Split up the given text in a useful way.

    This splits the text into individual tokens. The returned vaule is a 
    generator that yields `(line_num, token)` pairs. `line_num` is 1-indexed.
    '''

    for line_num, line in enumerate(text.split('\n')):
        for token in split_string(line, *_resources.NWS_DELIMITERS):
            yield (line_num + 1, token)
Example #2
0
 def test_split_string(self):
     s = 'abcdefg'
     self.assertEqual(prosl_utils.split_string(s), [s])
     self.assertEqual(prosl_utils.split_string(s, ' '), [s])
     self.assertEqual(prosl_utils.split_string(s, 
                                                     split_whitespace=False),
                      [s])
     ###
     s = 'abc  defg'
     self.assertEqual(prosl_utils.split_string(s), ['abc', 'defg'])
     self.assertEqual(prosl_utils.split_string(s, ' '), 
                      ['abc', 'defg'])
     self.assertEqual(prosl_utils.split_string(s,' ',split_whitespace=False), 
                      ['abc', '', 'defg'])
     ###
     s = 'a b`c!d--e><f\\g  '
     self.assertEqual(prosl_utils.split_string(s),
                      ['a', 'b`c!d--e><f\\g'])
     self.assertEqual(prosl_utils.split_string(s, '`', '--', '\\'),
                      ['a', 'b', 'c!d', 'e><f', 'g'])
     self.assertEqual(prosl_utils.split_string(s, '><', 
                                                     split_whitespace=False),
                      ['a b`c!d--e', 'f\\g  '])
     ###
     s = ''
     self.assertEqual(prosl_utils.split_string(s), [])
     self.assertEqual(prosl_utils.split_string(s, ' '), [])
     self.assertEqual(prosl_utils.split_string(s, split_whitespace=False),[])
     self.assertEqual(prosl_utils.split_string(s, 'a', 'b'), [])