def test_handler_string_result(self):
import _codecs
def f(exc):
return (b'foo', exc.end)
_codecs.register_error("test.test_codecs_not_a_string", f)
result = '\u1234'.encode('ascii', 'test.test_codecs_not_a_string')
assert result == b'foo'
def test_bad_handler_string_result(self):
import _codecs
def f(exc):
return ('foo', exc.end)
_codecs.register_error("test.test_codecs_not_a_string", f)
raises(TypeError, u'\u1234'.encode, 'ascii',
'test.test_codecs_not_a_string')
def _register_existing_errors():
import _codecs
_codecs.register_error("strict", strict_errors)
_codecs.register_error("ignore", ignore_errors)
_codecs.register_error("replace", replace_errors)
_codecs.register_error("xmlcharrefreplace", xmlcharrefreplace_errors)
_codecs.register_error("backslashreplace", backslashreplace_errors)
def test_bad_handler_string_result(self):
import _codecs
def f(exc):
return ('foo', exc.end)
_codecs.register_error("test.test_codecs_not_a_string", f)
raises(TypeError, u'\u1234'.encode, 'ascii',
'test.test_codecs_not_a_string')
def test_callback_None_index(self):
import _multibytecodec, _codecs
codec = _multibytecodec.__getcodec('cp932')
def myreplace(exc):
return ('x', None)
_codecs.register_error("test.cjktest", myreplace)
raises(TypeError, codec.encode, '\udeee', 'test.cjktest')
def test_bad_handler_string_result(self):
import _codecs
def f(exc):
return ("foo", exc.end)
_codecs.register_error("test.test_codecs_not_a_string", f)
raises(TypeError, u"\u1234".encode, "ascii", "test.test_codecs_not_a_string")
def test_callback_forward_index(self):
import _multibytecodec, _codecs
codec = _multibytecodec.__getcodec('cp932')
def myreplace(exc):
return ('REPLACED', exc.end + 2)
_codecs.register_error("test.cjktest", myreplace)
assert (codec.encode('abcd' + '\udeee' + 'efgh', 'test.cjktest') ==
(b'abcdREPLACEDgh', 9))
def test_unicode_error():
from _codecs import register_error
def handler(ex):
AreEqual(ex.object, u'\uac00')
return (u"", ex.end)
register_error("test_unicode_error", handler)
for mode in ip_supported_encodings: unichr(0xac00).encode(mode, "test_unicode_error")
def test_unicode_error():
from _codecs import register_error
def handler(ex):
AreEqual(ex.object, u'\uac00')
return (u"", ex.end)
register_error("test_unicode_error", handler)
for mode in ip_supported_encodings: unichr(0xac00).encode(mode, "test_unicode_error")
def test_handler_string_result(self):
import _codecs
def f(exc):
return (b'foo', exc.end)
_codecs.register_error("test.test_codecs_not_a_string", f)
result = '\u1234'.encode('ascii', 'test.test_codecs_not_a_string')
assert result == b'foo'
def test_unicode_error(self):
from iptest.misc_util import ip_supported_encodings
from _codecs import register_error
def handler(ex):
self.assertEqual(ex.object, u'\uac00')
return (u"", ex.end)
register_error("test_unicode_error", handler)
for mode in ip_supported_encodings: unichr(0xac00).encode(mode, "test_unicode_error")
def test_lookup_error():
'''
'''
#sanity
AssertError(LookupError, codecs.lookup_error, "blah garbage xyz")
def garbage_error1(someError): pass
codecs.register_error("blah garbage xyz", garbage_error1)
AreEqual(codecs.lookup_error("blah garbage xyz"), garbage_error1)
def garbage_error2(someError): pass
codecs.register_error("some other", garbage_error2)
AreEqual(codecs.lookup_error("some other"), garbage_error2)
def test_lookup_error():
'''
'''
#sanity
AssertError(LookupError, codecs.lookup_error, "blah garbage xyz")
def garbage_error1(someError): pass
codecs.register_error("blah garbage xyz", garbage_error1)
AreEqual(codecs.lookup_error("blah garbage xyz"), garbage_error1)
def garbage_error2(someError): pass
codecs.register_error("some other", garbage_error2)
AreEqual(codecs.lookup_error("some other"), garbage_error2)
def test_callback_backward_index(self):
import _multibytecodec, _codecs
codec = _multibytecodec.__getcodec('cp932')
def myreplace(exc):
if myreplace.limit > 0:
myreplace.limit -= 1
return ('REPLACED', 0)
else:
return ('TERMINAL', exc.end)
myreplace.limit = 3
_codecs.register_error("test.cjktest", myreplace)
assert (codec.encode('abcd' + '\udeee' + 'efgh', 'test.cjktest') ==
(b'abcdREPLACEDabcdREPLACEDabcdREPLACEDabcdTERMINALefgh', 9))
def test_last_byte_handler(self):
# issue bb-2389
import _codecs
_codecs.register_error('custom_replace', lambda exc:
(u'\ufffd', exc.start + 1))
for s, res in ((b"WORD\xe3\xab", (u'WORD\ufffd\ufffd', u'WORD\ufffd')),
(b"\xef\xbb\xbfWORD\xe3\xabWORD2",
(u'\ufeffWORD\ufffd\ufffdWORD2',
u'\ufeffWORD\ufffdWORD2'))):
r = s.decode('utf8', 'replace')
assert r == res[1]
r = s.decode('utf8', 'custom_replace')
assert r == res[0]
def test_register_error():
'''
TODO: test that these are actually used.
'''
#Sanity
def garbage_error0(): print "garbage_error0"
def garbage_error1(param1): print "garbage_error1:", param1
def garbage_error2(param1, param2): print "garbage_error2:", param1, "; ", param2
codecs.register_error("garbage0", garbage_error0)
codecs.register_error("garbage1", garbage_error1)
codecs.register_error("garbage2", garbage_error2)
codecs.register_error("garbage1dup", garbage_error1)
def test_register_error():
'''
TODO: test that these are actually used.
'''
#Sanity
def garbage_error0(): print "garbage_error0"
def garbage_error1(param1): print "garbage_error1:", param1
def garbage_error2(param1, param2): print "garbage_error2:", param1, "; ", param2
codecs.register_error("garbage0", garbage_error0)
codecs.register_error("garbage1", garbage_error1)
codecs.register_error("garbage2", garbage_error2)
codecs.register_error("garbage1dup", garbage_error1)
def test_errorhandler_collection(self):
import _codecs
errors = []
def record_error(exc):
if not isinstance(exc, UnicodeEncodeError):
raise TypeError("don't know how to handle %r" % exc)
errors.append(exc.object[exc.start:exc.end])
return (u'', exc.end)
_codecs.register_error("test.record", record_error)
sin = u"\xac\u1234\u1234\u20ac\u8000"
assert sin.encode("ascii", "test.record") == ""
assert errors == [sin]
errors = []
assert sin.encode("latin-1", "test.record") == "\xac"
assert errors == [u'\u1234\u1234\u20ac\u8000']
errors = []
assert sin.encode("iso-8859-15", "test.record") == "\xac\xa4"
assert errors == [u'\u1234\u1234', u'\u8000']
res += str(ord(ch))
res += ';'
return u''.join(res), exc.end
else:
raise TypeError("don't know how to handle %.400s in error callback"%type(exc))
def backslashreplace_errors(exc):
if isinstance(exc, UnicodeEncodeError):
p = []
for c in exc.object[exc.start:exc.end]:
p += '\\'
oc = ord(c)
if (oc >= 0x00010000):
p += 'U'
p += "%.8x" % ord(c)
elif (oc >= 0x100):
p += 'u'
p += "%.4x" % ord(c)
else:
p += 'x'
p += "%.2x" % ord(c)
return u''.join(p), exc.end
else:
raise TypeError("don't know how to handle %.400s in error callback"%type(exc))
register_error("strict", strict_errors)
register_error("ignore", ignore_errors)
register_error("replace", replace_errors)
register_error("xmlcharrefreplace", xmlcharrefreplace_errors)
register_error("backslashreplace", backslashreplace_errors)
raise TypeError("don't know how to handle %.400s in error callback" %
type(exc))
def backslashreplace_errors(exc):
if isinstance(exc, UnicodeEncodeError):
p = []
for c in exc.object[exc.start:exc.end]:
p += '\\'
oc = ord(c)
if (oc >= 0x00010000):
p += 'U'
p += "%.8x" % ord(c)
elif (oc >= 0x100):
p += 'u'
p += "%.4x" % ord(c)
else:
p += 'x'
p += "%.2x" % ord(c)
return u''.join(p), exc.end
else:
raise TypeError("don't know how to handle %.400s in error callback" %
type(exc))
register_error("strict", strict_errors)
register_error("ignore", ignore_errors)
register_error("replace", replace_errors)
register_error("xmlcharrefreplace", xmlcharrefreplace_errors)
register_error("backslashreplace", backslashreplace_errors)
