def guess_encoding_to_xml(string, output_encoding='utf-8', attrib=False,
control_chars='replace'):
'''Return a byte :class:`bytes` suitable for inclusion in xml
:arg string: :class:`str` or byte :class:`bytes` to be transformed into
a byte :class:`bytes` suitable for inclusion in xml. If string is
a byte :class:`bytes` we attempt to guess the encoding. If we cannot guess,
we fallback to ``latin-1``.
:kwarg output_encoding: Output encoding for the byte :class:`bytes`. This
should match the encoding of your xml file.
:kwarg attrib: If :data:`True`, escape the item for use in an xml
attribute. If :data:`False` (default) escape the item for use in
a text node.
:returns: :term:`utf-8` encoded byte :class:`bytes`
'''
# Unicode strings can just be run through unicode_to_xml()
if isunicodestring(string):
return unicode_to_xml(string, encoding=output_encoding,
attrib=attrib, control_chars=control_chars)
# Guess the encoding of the byte strings
input_encoding = guess_encoding(string)
# Return the new byte string
return byte_string_to_xml(string, input_encoding=input_encoding,
errors='replace', output_encoding=output_encoding,
attrib=attrib, control_chars=control_chars)
def utf8_width_chop(msg, chop=None):
'''**Deprecated** Return a string chopped to a given :term:`textual width`
Use :func:`~kitchen.text.display.textual_width_chop` and
:func:`~kitchen.text.display.textual_width` instead::
>>> msg = 'く ku ら ra と to み mi'
>>> # Old way:
>>> utf8_width_chop(msg, 5)
(5, 'く ku')
>>> # New way
>>> from kitchen.text.converters import to_bytes
>>> from kitchen.text.display import textual_width, textual_width_chop
>>> (textual_width(msg), to_bytes(textual_width_chop(msg, 5)))
(5, 'く ku')
'''
warnings.warn('kitchen.text.utf8.utf8_width_chop is deprecated. Use'
' kitchen.text.display.textual_width_chop instead', DeprecationWarning,
stacklevel=2)
if chop == None:
return textual_width(msg), msg
as_bytes = not isunicodestring(msg)
chopped_msg = textual_width_chop(msg, chop)
if as_bytes:
chopped_msg = to_bytes(chopped_msg)
return textual_width(chopped_msg), chopped_msg
def utf8_text_wrap(text, width=70, initial_indent='', subsequent_indent=''):
'''**Deprecated** Similar to :func:`textwrap.wrap` but understands
:term:`utf-8` data and doesn't screw up lists/blocks/etc
Use :func:`kitchen.text.display.wrap` instead
'''
warnings.warn('kitchen.text.utf8.utf8_text_wrap is deprecated. Use'
' kitchen.text.display.wrap instead',
DeprecationWarning, stacklevel=2)
as_bytes = not isunicodestring(text)
text = to_unicode(text)
lines = wrap(text, width=width, initial_indent=initial_indent,
subsequent_indent=subsequent_indent)
if as_bytes:
lines = [to_bytes(m) for m in lines]
return lines
def test_isunicodestring(self):
tools.assert_false(misc.isunicodestring('abc'))
tools.assert_true(misc.isunicodestring(u'abc'))
tools.assert_false(misc.isunicodestring(5))
def test_isunicodestring(self):
tools.assert_false(misc.isunicodestring(b'abc'))
tools.assert_true(misc.isunicodestring('abc'))
tools.assert_false(misc.isunicodestring(5))
def to_unicode(obj, encoding='utf-8', errors='replace', nonstring=None,
non_string=None):
'''Convert an object into a :class:`str` string
:arg obj: Object to convert to a :class:`str` string. This should
normally be a byte :class:`bytes`
:kwarg encoding: What encoding to try converting the byte :class:`bytes` as.
Defaults to :term:`utf-8`
:kwarg errors: If errors are found while decoding, perform this action.
Defaults to ``replace`` which replaces the invalid bytes with
a character that means the bytes were unable to be decoded. Other
values are the same as the error handling schemes in the `codec base
classes
<http://docs.python.org/library/codecs.html#codec-base-classes>`_.
For instance ``strict`` which raises an exception and ``ignore`` which
simply omits the non-decodable characters.
:kwarg nonstring: How to treat nonstring values. Possible values are:
:simplerepr: Attempt to call the object's "simple representation"
method and return that value. Python-2.3+ has two methods that
try to return a simple representation: :meth:`object.__unicode__`
and :meth:`object.__str__`. We first try to get a usable value
from :meth:`object.__unicode__`. If that fails we try the same
with :meth:`object.__str__`.
:empty: Return an empty :class:`str` string
:strict: Raise a :exc:`TypeError`
:passthru: Return the object unchanged
:repr: Attempt to return a :class:`str` string of the repr of the
object
Default is ``simplerepr``
:kwarg non_string: *Deprecated* Use :attr:`nonstring` instead
:raises TypeError: if :attr:`nonstring` is ``strict`` and
a non-:class:`basestring` object is passed in or if :attr:`nonstring`
is set to an unknown value
:raises UnicodeDecodeError: if :attr:`errors` is ``strict`` and
:attr:`obj` is not decodable using the given encoding
:returns: :class:`str` string or the original object depending on the
value of :attr:`nonstring`.
Usually this should be used on a byte :class:`bytes` but it can take both
byte :class:`bytes` and :class:`str` strings intelligently. Nonstring
objects are handled in different ways depending on the setting of the
:attr:`nonstring` parameter.
The default values of this function are set so as to always return
a :class:`str` string and never raise an error when converting from
a byte :class:`bytes` to a :class:`str` string. However, when you do
not pass validly encoded text (or a nonstring object), you may end up with
output that you don't expect. Be sure you understand the requirements of
your data, not just ignore errors by passing it through this function.
.. versionchanged:: 0.2.1a2
Deprecated :attr:`non_string` in favor of :attr:`nonstring` parameter and changed
default value to ``simplerepr``
'''
# Could use isbasestring/isunicode here but we want this code to be as
# fast as possible
if isinstance(obj, str):
return obj
if isinstance(obj, (bytes, bytearray)):
if encoding in _UTF8_ALIASES:
return str(obj, 'utf-8', errors)
if encoding in _LATIN1_ALIASES:
return obj.decode('latin-1', errors)
return obj.decode(encoding, errors)
if non_string:
warnings.warn('non_string is a deprecated parameter of'
' to_unicode(). Use nonstring instead', DeprecationWarning,
stacklevel=2)
if not nonstring:
nonstring = non_string
if not nonstring:
nonstring = 'simplerepr'
if nonstring == 'empty':
return ''
elif nonstring == 'passthru':
return obj
elif nonstring == 'simplerepr':
try:
simple = str(obj)
except UnicodeError:
try:
simple = obj.__str__()
except (UnicodeError, AttributeError):
simple = ''
if not isunicodestring(simple):
return str(simple, encoding, errors)
return simple
elif nonstring in ('repr', 'strict'):
obj_repr = repr(obj)
if not isunicodestring(obj_repr):
obj_repr = str(obj_repr, encoding, errors)
if nonstring == 'repr':
return obj_repr
raise TypeError('to_unicode was given "%(obj)s" which is neither'
' a byte string (str) or a unicode string' %
{'obj': obj_repr.encode(encoding, 'replace')})
raise TypeError('nonstring value, %(param)s, is not set to a valid'
' action' % {'param': nonstring})
def to_bytes(obj, encoding='utf-8', errors='replace', nonstring=None,
non_string=None):
'''Convert an object into a byte :class:`str`
:arg obj: Object to convert to a byte :class:`str`. This should normally
be a :class:`unicode` string.
:kwarg encoding: Encoding to use to convert the :class:`unicode` string
into a byte :class:`str`. Defaults to :term:`utf-8`.
:kwarg errors: If errors are found while encoding, perform this action.
Defaults to ``replace`` which replaces the invalid bytes with
a character that means the bytes were unable to be encoded. Other
values are the same as the error handling schemes in the `codec base
classes
<http://docs.python.org/library/codecs.html#codec-base-classes>`_.
For instance ``strict`` which raises an exception and ``ignore`` which
simply omits the non-encodable characters.
:kwarg nonstring: How to treat nonstring values. Possible values are:
:simplerepr: Attempt to call the object's "simple representation"
method and return that value. Python-2.3+ has two methods that
try to return a simple representation: :meth:`object.__unicode__`
and :meth:`object.__str__`. We first try to get a usable value
from :meth:`object.__str__`. If that fails we try the same
with :meth:`object.__unicode__`.
:empty: Return an empty byte :class:`str`
:strict: Raise a :exc:`TypeError`
:passthru: Return the object unchanged
:repr: Attempt to return a byte :class:`str` of the :func:`repr` of the
object
Default is ``simplerepr``.
:kwarg non_string: *Deprecated* Use :attr:`nonstring` instead.
:raises TypeError: if :attr:`nonstring` is ``strict`` and
a non-:class:`basestring` object is passed in or if :attr:`nonstring`
is set to an unknown value.
:raises UnicodeEncodeError: if :attr:`errors` is ``strict`` and all of the
bytes of :attr:`obj` are unable to be encoded using :attr:`encoding`.
:returns: byte :class:`str` or the original object depending on the value
of :attr:`nonstring`.
.. warning::
If you pass a byte :class:`str` into this function the byte
:class:`str` is returned unmodified. It is **not** re-encoded with
the specified :attr:`encoding`. The easiest way to achieve that is::
to_bytes(to_unicode(text), encoding='utf-8')
The initial :func:`to_unicode` call will ensure text is
a :class:`unicode` string. Then, :func:`to_bytes` will turn that into
a byte :class:`str` with the specified encoding.
Usually, this should be used on a :class:`unicode` string but it can take
either a byte :class:`str` or a :class:`unicode` string intelligently.
Nonstring objects are handled in different ways depending on the setting
of the :attr:`nonstring` parameter.
The default values of this function are set so as to always return a byte
:class:`str` and never raise an error when converting from unicode to
bytes. However, when you do not pass an encoding that can validly encode
the object (or a non-string object), you may end up with output that you
don't expect. Be sure you understand the requirements of your data, not
just ignore errors by passing it through this function.
.. versionchanged:: 0.2.1a2
Deprecated :attr:`non_string` in favor of :attr:`nonstring` parameter
and changed default value to ``simplerepr``
'''
# Could use isbasestring, isbytestring here but we want this to be as fast
# as possible
if isinstance(obj, basestring):
if isinstance(obj, str):
return obj
return obj.encode(encoding, errors)
if non_string:
warnings.warn('non_string is a deprecated parameter of'
' to_bytes(). Use nonstring instead', DeprecationWarning,
stacklevel=2)
if not nonstring:
nonstring = non_string
if not nonstring:
nonstring = 'simplerepr'
if nonstring == 'empty':
return ''
elif nonstring == 'passthru':
return obj
elif nonstring == 'simplerepr':
try:
simple = str(obj)
except UnicodeError:
try:
simple = obj.__str__()
except (AttributeError, UnicodeError):
simple = None
if not simple:
try:
simple = obj.__unicode__()
except (AttributeError, UnicodeError):
simple = ''
if isunicodestring(simple):
simple = simple.encode(encoding, 'replace')
return simple
elif nonstring in ('repr', 'strict'):
try:
obj_repr = obj.__repr__()
except (AttributeError, UnicodeError):
obj_repr = ''
if isunicodestring(obj_repr):
obj_repr = obj_repr.encode(encoding, errors)
else:
obj_repr = str(obj_repr)
if nonstring == 'repr':
return obj_repr
raise TypeError('to_bytes was given "%(obj)s" which is neither'
' a unicode string or a byte string (str)' % {'obj': obj_repr})
raise TypeError('nonstring value, %(param)s, is not set to a valid'
' action' % {'param': nonstring})