def _preprocess_chars(chars: str, do_compress_char_ranges: bool = False):
chars = process_cldr_chars(chars)
if do_compress_char_ranges:
chars = compress_char_ranges(chars)
return chars
def test_process_cldr_chars_5(self):
chars = utils.process_cldr_chars(r'[]')
assert chars == []
def test_process_cldr_chars_4(self):
chars = utils.process_cldr_chars(
'[a á à â ǎ ā {a\u1DC6}{a\u1DC7} b ɓ]')
assert 'a\u1DC6a\u1DC7' in chars
def test_process_cldr_chars_3(self):
chars = utils.process_cldr_chars(
r"[\- ‐ – — , ; \: ! ¡ ? ¿ . … ' ‘ ’ " "
r"“ ” « » ( ) \[ \] § @ * / \\ \& # † ‡ ′ ″]")
assert '\\' in chars
def test_process_cldr_chars_2(self):
chars = utils.process_cldr_chars(
r'[‐ – , ፡ ፣ ፤ ፥ ፦ ! ? . ። ‹ › « » ( ) \[ \]]')
assert '[' in chars
assert ']' in chars
def test_process_cldr_chars(self):
assert utils.process_cldr_chars(r'[ \- а-е щ {а\u0301}]') == [
' ', '-', 'а', 'а́', 'б', 'в', 'г', 'д', 'е', 'щ'
]