def test_isbytestring(self):
tools.assert_true(misc.isbytestring('abc'))
tools.assert_false(misc.isbytestring(u'abc'))
tools.assert_false(misc.isbytestring(5))
def byte_string_to_xml(byte_string, input_encoding='utf-8', errors='replace',
output_encoding='utf-8', attrib=False, control_chars='replace'):
'''Make sure a byte :class:`bytes` is validly encoded for xml output
:arg byte_string: Byte :class:`bytes` to turn into valid xml output
:kwarg input_encoding: Encoding of :attr:`byte_string`. Default ``utf-8``
:kwarg errors: How to handle errors encountered while decoding the
:attr:`byte_string` into :class:`str` at the beginning of the
process. Values are:
:replace: (default) Replace the invalid bytes with a ``?``
:ignore: Remove the characters altogether from the output
:strict: Raise an :exc:`UnicodeDecodeError` when we encounter
a non-decodable character
:kwarg output_encoding: Encoding for the xml file that this string will go
into. Default is ``utf-8``. If all the characters in
:attr:`byte_string` are not encodable in this encoding, the unknown
characters will be entered into the output string using xml character
references.
:kwarg attrib: If :data:`True`, quote the string for use in an xml
attribute. If :data:`False` (default), quote for use in an xml text
field.
:kwarg control_chars: XML does not allow :term:`control characters`. When
we encounter those we need to know what to do. Valid options are:
:replace: (default) Replace the :term:`control characters` with ``?``
:ignore: Remove the characters altogether from the output
:strict: Raise an error when we encounter a :term:`control character`
:raises XmlEncodeError: If :attr:`control_chars` is set to ``strict`` and
the string to be made suitable for output to xml contains
:term:`control characters` then we raise this exception.
:raises UnicodeDecodeError: If errors is set to ``strict`` and the
:attr:`byte_string` contains bytes that are not decodable using
:attr:`input_encoding`, this error is raised
:rtype: byte :class:`bytes`
:returns: representation of the byte :class:`bytes` in the output encoding with
any bytes that aren't available in xml taken care of.
Use this when you have a byte :class:`bytes` representing text that you need
to make suitable for output to xml. There are several cases where this
is the case. For instance, if you need to transform some strings encoded
in ``latin-1`` to :term:`utf-8` for output::
utf8_string = byte_string_to_xml(latin1_string, input_encoding='latin-1')
If you already have strings in the proper encoding you may still want to
use this function to remove :term:`control characters`::
cleaned_string = byte_string_to_xml(string, input_encoding='utf-8', output_encoding='utf-8')
.. seealso::
:func:`unicode_to_xml`
for other ideas on using this function
'''
if not isbytestring(byte_string):
raise XmlEncodeError('byte_string_to_xml can only take a byte'
' string as its first argument. Use unicode_to_xml for'
' unicode (str) strings')
# Decode the string into unicode
u_string = str(byte_string, input_encoding, errors)
return unicode_to_xml(u_string, encoding=output_encoding,
attrib=attrib, control_chars=control_chars)
def test_isbytestring(self):
tools.assert_true(misc.isbytestring(b'abc'))
tools.assert_false(misc.isbytestring('abc'))
tools.assert_false(misc.isbytestring(5))
def to_unicode(obj, encoding='utf-8', errors='replace', nonstring=None,
non_string=None):
'''Convert an object into a :class:`unicode` string
:arg obj: Object to convert to a :class:`unicode` string. This should
normally be a byte :class:`str`
:kwarg encoding: What encoding to try converting the byte :class:`str` as.
Defaults to :term:`utf-8`
:kwarg errors: If errors are found while decoding, perform this action.
Defaults to ``replace`` which replaces the invalid bytes with
a character that means the bytes were unable to be decoded. Other
values are the same as the error handling schemes in the `codec base
classes
<http://docs.python.org/library/codecs.html#codec-base-classes>`_.
For instance ``strict`` which raises an exception and ``ignore`` which
simply omits the non-decodable characters.
:kwarg nonstring: How to treat nonstring values. Possible values are:
:simplerepr: Attempt to call the object's "simple representation"
method and return that value. Python-2.3+ has two methods that
try to return a simple representation: :meth:`object.__unicode__`
and :meth:`object.__str__`. We first try to get a usable value
from :meth:`object.__unicode__`. If that fails we try the same
with :meth:`object.__str__`.
:empty: Return an empty :class:`unicode` string
:strict: Raise a :exc:`TypeError`
:passthru: Return the object unchanged
:repr: Attempt to return a :class:`unicode` string of the repr of the
object
Default is ``simplerepr``
:kwarg non_string: *Deprecated* Use :attr:`nonstring` instead
:raises TypeError: if :attr:`nonstring` is ``strict`` and
a non-:class:`basestring` object is passed in or if :attr:`nonstring`
is set to an unknown value
:raises UnicodeDecodeError: if :attr:`errors` is ``strict`` and
:attr:`obj` is not decodable using the given encoding
:returns: :class:`unicode` string or the original object depending on the
value of :attr:`nonstring`.
Usually this should be used on a byte :class:`str` but it can take both
byte :class:`str` and :class:`unicode` strings intelligently. Nonstring
objects are handled in different ways depending on the setting of the
:attr:`nonstring` parameter.
The default values of this function are set so as to always return
a :class:`unicode` string and never raise an error when converting from
a byte :class:`str` to a :class:`unicode` string. However, when you do
not pass validly encoded text (or a nonstring object), you may end up with
output that you don't expect. Be sure you understand the requirements of
your data, not just ignore errors by passing it through this function.
.. versionchanged:: 0.2.1a2
Deprecated :attr:`non_string` in favor of :attr:`nonstring` parameter and changed
default value to ``simplerepr``
'''
# Could use isbasestring/isunicode here but we want this code to be as
# fast as possible
if isinstance(obj, basestring):
if isinstance(obj, unicode):
return obj
if encoding in _UTF8_ALIASES:
return unicode(obj, 'utf-8', errors)
if encoding in _LATIN1_ALIASES:
return unicode(obj, 'latin-1', errors)
return obj.decode(encoding, errors)
if non_string:
warnings.warn('non_string is a deprecated parameter of'
' to_unicode(). Use nonstring instead', DeprecationWarning,
stacklevel=2)
if not nonstring:
nonstring = non_string
if not nonstring:
nonstring = 'simplerepr'
if nonstring == 'empty':
return u''
elif nonstring == 'passthru':
return obj
elif nonstring == 'simplerepr':
try:
simple = obj.__unicode__()
except (AttributeError, UnicodeError):
simple = None
if not simple:
try:
simple = str(obj)
except UnicodeError:
try:
simple = obj.__str__()
except (UnicodeError, AttributeError):
simple = u''
if isbytestring(simple):
return unicode(simple, encoding, errors)
return simple
elif nonstring in ('repr', 'strict'):
obj_repr = repr(obj)
if isbytestring(obj_repr):
obj_repr = unicode(obj_repr, encoding, errors)
if nonstring == 'repr':
return obj_repr
raise TypeError('to_unicode was given "%(obj)s" which is neither'
' a byte string (str) or a unicode string' %
{'obj': obj_repr.encode(encoding, 'replace')})
raise TypeError('nonstring value, %(param)s, is not set to a valid'
' action' % {'param': nonstring})
