def test_isbytestring(self): tools.assert_true(misc.isbytestring('abc')) tools.assert_false(misc.isbytestring(u'abc')) tools.assert_false(misc.isbytestring(5))
def byte_string_to_xml(byte_string, input_encoding='utf-8', errors='replace', output_encoding='utf-8', attrib=False, control_chars='replace'): '''Make sure a byte :class:`bytes` is validly encoded for xml output :arg byte_string: Byte :class:`bytes` to turn into valid xml output :kwarg input_encoding: Encoding of :attr:`byte_string`. Default ``utf-8`` :kwarg errors: How to handle errors encountered while decoding the :attr:`byte_string` into :class:`str` at the beginning of the process. Values are: :replace: (default) Replace the invalid bytes with a ``?`` :ignore: Remove the characters altogether from the output :strict: Raise an :exc:`UnicodeDecodeError` when we encounter a non-decodable character :kwarg output_encoding: Encoding for the xml file that this string will go into. Default is ``utf-8``. If all the characters in :attr:`byte_string` are not encodable in this encoding, the unknown characters will be entered into the output string using xml character references. :kwarg attrib: If :data:`True`, quote the string for use in an xml attribute. If :data:`False` (default), quote for use in an xml text field. :kwarg control_chars: XML does not allow :term:`control characters`. When we encounter those we need to know what to do. Valid options are: :replace: (default) Replace the :term:`control characters` with ``?`` :ignore: Remove the characters altogether from the output :strict: Raise an error when we encounter a :term:`control character` :raises XmlEncodeError: If :attr:`control_chars` is set to ``strict`` and the string to be made suitable for output to xml contains :term:`control characters` then we raise this exception. :raises UnicodeDecodeError: If errors is set to ``strict`` and the :attr:`byte_string` contains bytes that are not decodable using :attr:`input_encoding`, this error is raised :rtype: byte :class:`bytes` :returns: representation of the byte :class:`bytes` in the output encoding with any bytes that aren't available in xml taken care of. Use this when you have a byte :class:`bytes` representing text that you need to make suitable for output to xml. There are several cases where this is the case. For instance, if you need to transform some strings encoded in ``latin-1`` to :term:`utf-8` for output:: utf8_string = byte_string_to_xml(latin1_string, input_encoding='latin-1') If you already have strings in the proper encoding you may still want to use this function to remove :term:`control characters`:: cleaned_string = byte_string_to_xml(string, input_encoding='utf-8', output_encoding='utf-8') .. seealso:: :func:`unicode_to_xml` for other ideas on using this function ''' if not isbytestring(byte_string): raise XmlEncodeError('byte_string_to_xml can only take a byte' ' string as its first argument. Use unicode_to_xml for' ' unicode (str) strings') # Decode the string into unicode u_string = str(byte_string, input_encoding, errors) return unicode_to_xml(u_string, encoding=output_encoding, attrib=attrib, control_chars=control_chars)
def test_isbytestring(self): tools.assert_true(misc.isbytestring(b'abc')) tools.assert_false(misc.isbytestring('abc')) tools.assert_false(misc.isbytestring(5))
def to_unicode(obj, encoding='utf-8', errors='replace', nonstring=None, non_string=None): '''Convert an object into a :class:`unicode` string :arg obj: Object to convert to a :class:`unicode` string. This should normally be a byte :class:`str` :kwarg encoding: What encoding to try converting the byte :class:`str` as. Defaults to :term:`utf-8` :kwarg errors: If errors are found while decoding, perform this action. Defaults to ``replace`` which replaces the invalid bytes with a character that means the bytes were unable to be decoded. Other values are the same as the error handling schemes in the `codec base classes <http://docs.python.org/library/codecs.html#codec-base-classes>`_. For instance ``strict`` which raises an exception and ``ignore`` which simply omits the non-decodable characters. :kwarg nonstring: How to treat nonstring values. Possible values are: :simplerepr: Attempt to call the object's "simple representation" method and return that value. Python-2.3+ has two methods that try to return a simple representation: :meth:`object.__unicode__` and :meth:`object.__str__`. We first try to get a usable value from :meth:`object.__unicode__`. If that fails we try the same with :meth:`object.__str__`. :empty: Return an empty :class:`unicode` string :strict: Raise a :exc:`TypeError` :passthru: Return the object unchanged :repr: Attempt to return a :class:`unicode` string of the repr of the object Default is ``simplerepr`` :kwarg non_string: *Deprecated* Use :attr:`nonstring` instead :raises TypeError: if :attr:`nonstring` is ``strict`` and a non-:class:`basestring` object is passed in or if :attr:`nonstring` is set to an unknown value :raises UnicodeDecodeError: if :attr:`errors` is ``strict`` and :attr:`obj` is not decodable using the given encoding :returns: :class:`unicode` string or the original object depending on the value of :attr:`nonstring`. Usually this should be used on a byte :class:`str` but it can take both byte :class:`str` and :class:`unicode` strings intelligently. Nonstring objects are handled in different ways depending on the setting of the :attr:`nonstring` parameter. The default values of this function are set so as to always return a :class:`unicode` string and never raise an error when converting from a byte :class:`str` to a :class:`unicode` string. However, when you do not pass validly encoded text (or a nonstring object), you may end up with output that you don't expect. Be sure you understand the requirements of your data, not just ignore errors by passing it through this function. .. versionchanged:: 0.2.1a2 Deprecated :attr:`non_string` in favor of :attr:`nonstring` parameter and changed default value to ``simplerepr`` ''' # Could use isbasestring/isunicode here but we want this code to be as # fast as possible if isinstance(obj, basestring): if isinstance(obj, unicode): return obj if encoding in _UTF8_ALIASES: return unicode(obj, 'utf-8', errors) if encoding in _LATIN1_ALIASES: return unicode(obj, 'latin-1', errors) return obj.decode(encoding, errors) if non_string: warnings.warn('non_string is a deprecated parameter of' ' to_unicode(). Use nonstring instead', DeprecationWarning, stacklevel=2) if not nonstring: nonstring = non_string if not nonstring: nonstring = 'simplerepr' if nonstring == 'empty': return u'' elif nonstring == 'passthru': return obj elif nonstring == 'simplerepr': try: simple = obj.__unicode__() except (AttributeError, UnicodeError): simple = None if not simple: try: simple = str(obj) except UnicodeError: try: simple = obj.__str__() except (UnicodeError, AttributeError): simple = u'' if isbytestring(simple): return unicode(simple, encoding, errors) return simple elif nonstring in ('repr', 'strict'): obj_repr = repr(obj) if isbytestring(obj_repr): obj_repr = unicode(obj_repr, encoding, errors) if nonstring == 'repr': return obj_repr raise TypeError('to_unicode was given "%(obj)s" which is neither' ' a byte string (str) or a unicode string' % {'obj': obj_repr.encode(encoding, 'replace')}) raise TypeError('nonstring value, %(param)s, is not set to a valid' ' action' % {'param': nonstring})