def test_trans_single_ignore(self): """Test that a single word with markers gets transliterated when ignore markers is set to True. """ trans = Transliterator(TEST_MAPPER, '@@') assert trans.transliterate(u'@@Hello', False, True) == u'@@Xxxxx'
def test_trans_single_strip(self): """Test that a single word with markers does not get transliterated but markers do get stripped when strip_markers is set to True. """ trans = Transliterator(TEST_MAPPER, '@@') assert trans.transliterate(u'@@Hello', True) == u'Hello'
def test_trans_single_ignore_strip(self): """Test that a single word with markers gets transliterated with markers stripped when both strip_markers and ignore_markers are set to True. """ trans = Transliterator(TEST_MAPPER, '@@') assert trans.transliterate(u'@@Hello', True, True) == u'Xxxxx'
def test_trans_sent_no_markers(self): """Test that a sentence with no markers gets transliterated. """ sent_orig = u'Hello World, this is a sentence!' sent_out = u'Xxxxx Xxxxx, xxxx xx x xxxxxxxx!' trans = Transliterator(TEST_MAPPER, '@@') assert trans.transliterate(sent_orig) == sent_out
def test_trans_sent_with_markers(self): """Test that tokens with markers in a sentence do not get transliterated. """ sent_orig = u'Hello @@World, this is a @@sentence!' sent_out = u'Xxxxx @@World, xxxx xx x @@sentence!' trans = Transliterator(TEST_MAPPER, '@@') assert trans.transliterate(sent_orig) == sent_out
def test_trans_sent_ignore(self): """Test that tokens with markers in a sentence get transliterated when ignore markers is set to True. """ sent_orig = u'Hello @@World, this is a @@sentence!' sent_out = u'Xxxxx @@Xxxxx, xxxx xx x @@xxxxxxxx!' trans = Transliterator(TEST_MAPPER, '@@') assert trans.transliterate(sent_orig, False, True) == sent_out
def test_trans_sent_ignore_strip(self): """Test that tokens with markers in a sentence get transliterated with markers stripped when both strip_markers and ignore_markers are set to True. """ sent_orig = u'Hello @@World, this is a @@sentence!' sent_out = u'Xxxxx Xxxxx, xxxx xx x xxxxxxxx!' trans = Transliterator(TEST_MAPPER, '@@') assert trans.transliterate(sent_orig, True, True) == sent_out
def test_trans_sent_strip(self): """Test that tokens with markers in a sentence do not get transliterated but markers do get stripped when strip_markers is set to True. """ sent_orig = u'Hello @@World, this is a @@sentence!' sent_out = u'Xxxxx World, xxxx xx x sentence!' trans = Transliterator(TEST_MAPPER, '@@') assert trans.transliterate(sent_orig, True) == sent_out
def test_init_empty_marker(self): """Test that init raises a ValueError when given a marker that is an empty string. """ with pytest.raises(ValueError): Transliterator(TEST_MAPPER, '')
def test_init_invalid_type_marker(self): """Test that init raises a TypeError when given a marker that is not a string. """ with pytest.raises(TypeError): Transliterator(TEST_MAPPER, [])
def test_init_invalid_type_mapper(self): """Test that init raises a TypeError when given a mapper that is not a CharMapper instance. """ with pytest.raises(TypeError): Transliterator({})
def test_init_invalid_marker2(self): """Test that init raises a ValueError when given an invalid marker ( whitespace at the end). """ with pytest.raises(ValueError): Transliterator(TEST_MAPPER, '@@LAT@@ ')
def test_init_none_marker(self): """Test that init raises a TypeError when given a marker that is None. """ with pytest.raises(TypeError): Transliterator(TEST_MAPPER, None)
def test_init_valid_mapper(self): """Test that init doesn't raise an error when given a valid mapper. """ assert Transliterator(TEST_MAPPER)
def test_trans_single_no_markers(self): """Test that a single word with no markers gets transliterated. """ trans = Transliterator(TEST_MAPPER, '@@') assert trans.transliterate(u'Hello') == u'Xxxxx'
def test_trans_single_with_markers(self): """Test that a single word with markers does not get transliterated. """ trans = Transliterator(TEST_MAPPER, '@@') assert trans.transliterate(u'@@Hello') == u'@@Hello'
def main(): # pragma: no cover try: version = ('CAMeL Tools v{}'.format(__version__)) arguments = docopt(__doc__, version=version) if arguments['--list']: for scheme in _BUILTIN_SCHEMES: print("{} {}".format(scheme[0].ljust(20), scheme[1])) sys.exit(0) if arguments['--scheme'] is not None: if arguments['--scheme'] not in [s[0] for s in _BUILTIN_SCHEMES]: sys.stderr.write('Error: {} is not a valid scheme.\n' 'Run `camel_transliterate -l` to see the list' ' of available schemes.' '\n'.format(repr(arguments['--scheme']))) sys.exit(1) if arguments['--marker'] is None: marker = '@@IGNORE@@' else: marker = arguments['--marker'] ignore_markers = arguments['--ignore-markers'] strip_markers = arguments['--strip-markers'] # Open files (or just use stdin and stdout) fin, fout = _open_files(arguments['FILE'], arguments['--output']) # Load the CharMapper and initialize a Transliterator with it try: mapper = CharMapper.builtin_mapper(arguments['--scheme']) trans = Transliterator(mapper, marker) except Exception: # pylint: disable=W0703 sys.stderr.write('Error: Could not load builtin scheme' ' {}.\n'.format(repr(arguments['--scheme']))) sys.exit(1) # Transliterate lines try: for line in fin: line = force_unicode(line) if six.PY3: fout.write( trans.transliterate(line, strip_markers, ignore_markers)) else: fout.write( force_encoding( trans.transliterate(line, strip_markers, ignore_markers))) fout.flush() # If everything worked so far, this shouldn't happen except Exception: # pylint: disable=W0703 sys.stderr.write('Error: An unkown error occured during ' 'transliteration.\n') sys.exit(1) # Cleanup if arguments['FILE'] is not None: fin.close() if arguments['--output'] is not None: fout.close() sys.exit(0) except KeyboardInterrupt: sys.stderr.write('Exiting...\n') sys.exit(1) except Exception: sys.stderr.write('Error: An unknown error occurred.\n') sys.exit(1)
def test_trans_empty(self): """Test that transliterating an empty string returns an empty string. """ trans = Transliterator(TEST_MAPPER, '@@') assert trans.transliterate(u'') == u''