def test_tokenize_none(): s = Separator(phone=None, syllable=None, word=' ') text = 'te9abesitosgr uNone' assert list(s.tokenize(text, level='word')) == ['te9abesitosgr', 'uNone'] assert s.strip('uNone') == 'uNone' assert s.strip('None') == 'None' assert s.strip('Noneu') == 'Noneu'
def test_strip(): s = Separator(phone='p', syllable='s', word='w') assert s.strip('') == '' assert s.strip('..') == '..' assert s.strip('p.p') == '.' assert s.strip('p .p') == '.' assert s.strip('psw p') == '' assert s.strip('ps w p') == '' assert s.strip(' pp ') == '' assert s.strip('sw p', level='phone') == 'sw' assert s.strip('sw p', level='syllable') == 'w p' assert s.strip('sw p', level='word') == 'sw p'