def test_replace_non_ascii(self):
data = ustr(b'\xef\xbb\xbfhehe', encoding='utf-8')
self.assertEqual('hehe', textutil.replace_non_ascii(data))
data = "abcd\xa0e\xf0fghijk\xbblm"
self.assertEqual("abcdefghijklm", textutil.replace_non_ascii(data))
data = "abcd\xa0e\xf0fghijk\xbblm"
self.assertEqual("abcdXeXfghijkXlm",
textutil.replace_non_ascii(data, replace_char='X'))
self.assertEqual('', textutil.replace_non_ascii(None))
def test_replace_non_ascii(self):
data = ustr(b'\xef\xbb\xbfhehe', encoding='utf-8')
self.assertEqual('hehe', textutil.replace_non_ascii(data))
data = "abcd\xa0e\xf0fghijk\xbblm"
self.assertEqual("abcdefghijklm", textutil.replace_non_ascii(data))
data = "abcd\xa0e\xf0fghijk\xbblm"
self.assertEqual("abcdXeXfghijkXlm",
textutil.replace_non_ascii(data, replace_char='X'))
self.assertEqual('', textutil.replace_non_ascii(None))
def read_response_error(resp):
result = ''
if resp is not None:
try:
result = "[HTTP Failed] [{0}: {1}] {2}".format(
resp.status, resp.reason, resp.read())
# this result string is passed upstream to several methods
# which do a raise HttpError() or a format() of some kind;
# as a result it cannot have any unicode characters
if PY_VERSION_MAJOR < 3:
result = ustr(result, encoding='ascii', errors='ignore')
else:
result = result\
.encode(encoding='ascii', errors='ignore')\
.decode(encoding='ascii', errors='ignore')
result = textutil.replace_non_ascii(result)
except Exception:
logger.warn(traceback.format_exc())
return result
def read_response_error(resp):
result = ''
if resp is not None:
try:
result = "[HTTP Failed] [{0}: {1}] {2}".format(
resp.status,
resp.reason,
resp.read())
# this result string is passed upstream to several methods
# which do a raise HttpError() or a format() of some kind;
# as a result it cannot have any unicode characters
if PY_VERSION_MAJOR < 3:
result = ustr(result, encoding='ascii', errors='ignore')
else:
result = result\
.encode(encoding='ascii', errors='ignore')\
.decode(encoding='ascii', errors='ignore')
result = textutil.replace_non_ascii(result)
except Exception:
logger.warn(traceback.format_exc())
return result