def test_should_not_split_word(self):
assert _tokenize_text('Abc') == ['Abc']
def test_should_split_word_on_lower_to_upper_case(self):
assert _tokenize_text('abcDEF') == ['abc', 'DEF']
def test_should_split_on_dot(self):
assert _tokenize_text(' .A. ') == [' ', '.', 'A', '.', ' ']
def test_should_brackets(self):
text = r' <{[(A)]}> '
assert _tokenize_text(text) == list(text)
def test_should_split_on_comma(self):
assert _tokenize_text(' ,A, ') == [' ', ',', 'A', ',', ' ']
def test_should_keep_tailing_space_of_item(self):
assert _tokenize_text('A ') == ['A', ' ']
def test_should_keep_preceding_space_of_item(self):
assert _tokenize_text(' A') == [' ', 'A']
def test_should_add_space_between_two_items(self):
assert _tokenize_text('A B') == ['A', ' ', 'B']
def test_should_not_add_space_to_single_item(self):
assert _tokenize_text('A') == ['A']
