def test_adapter_registered(self): """ test naaya.i18n's IPreferredCharset adapter is registered """ req = create_request() adapter = IUserPreferredCharsets(req) self.assertTrue(isinstance(adapter, NyHTTPCharsets)) self.assertEqual(adapter.getPreferredCharsets(), ['utf-8', 'iso-8859-1', '*'])
def _decode(self, text): """Try to decode the text using one of the available charsets.""" # All text comes from parse_qsl or multipart.parse_form_data, and # has already been decoded into Unicode using WSGI's privileged # encoding (ISO-8859-1). if self.charsets is None: envadapter = IUserPreferredCharsets(self) self.charsets = envadapter.getPreferredCharsets() or ['utf-8'] self.charsets = [c for c in self.charsets if c != '*'] if not isinstance(text, bytes): if self.charsets and self.charsets[0] == 'iso-8859-1': # optimization: we are trying to decode something already # decoded for us, let's just return it rather than waste # time decoding... return text # undo what parse_qsl/multipart.parse_form_data did and maintain # backwards compat text = text.encode('latin-1') for charset in self.charsets: try: text = text.decode(charset) break except UnicodeError: pass # XXX so when none of the provided charsets works we just return bytes # and let the application crash??? return text
def processInputs(request, charsets=None): """ Override Products.Five.browser.decode.processInputs """ if charsets is None: envadapter = IUserPreferredCharsets(request) charsets = envadapter.getPreferredCharsets() or ['utf-8'] for name, value in request.form.items(): if not (isCGI_NAME(name) or name.startswith('HTTP_')): # XXX => really dirty if name=='groups' or name=='users': request.form[name] = value elif isinstance(value, str): request.form[name] = _decode(value, charsets) elif isinstance(value, list): request.form[name] = [_decode(val, charsets) for val in value if isinstance(val, str)] elif isinstance(value, tuple): request.form[name] = tuple([_decode(val, charsets) for val in value if isinstance(val, str)])
def _decode(self, text): """Try to decode the text using one of the available charsets.""" # All text comes from cgi.FieldStorage. On Python 2 it's all bytes # and we must decode. On Python 3 it's already been decoded into # Unicode, using the charset we specified when instantiating the # FieldStorage instance (Latin-1). if self.charsets is None: envadapter = IUserPreferredCharsets(self) self.charsets = envadapter.getPreferredCharsets() or ['utf-8'] self.charsets = [c for c in self.charsets if c != '*'] if not PYTHON2 and not isinstance(text, bytes): if self.charsets and self.charsets[0] == 'iso-8859-1': # optimization: we are trying to decode something # cgi.FieldStorage already decoded for us, let's just return it # rather than waste time decoding... return text # undo what cgi.FieldStorage did and maintain backwards compat text = text.encode('latin-1') for charset in self.charsets: try: text = text.decode(charset) break except UnicodeError: pass # XXX so when none of the provided charsets works we just return bytes # and let the application crash??? return text
def processInputs(request, charsets=None): """Process the values in request.form to decode strings to unicode, using the passed-in list of charsets. If none are passed in, look up the user's preferred charsets. The default is to use utf-8. """ if IProcessedRequest.providedBy(request): return if charsets is None: envadapter = IUserPreferredCharsets(request, None) if envadapter is None: charsets = ['utf-8'] else: charsets = envadapter.getPreferredCharsets() or ['utf-8'] for name, value in request.form.items(): if not (isCGI_NAME(name) or name.startswith('HTTP_')): if isinstance(value, str): request.form[name] = _decode(value, charsets) elif isinstance(value, (list, tuple,)): newValue = [] for val in value: if isinstance(val, str): val = _decode(val, charsets) newValue.append(val) if isinstance(value, tuple): newValue = tuple(value) request.form[name] = newValue interface.alsoProvides(request, IProcessedRequest)
def _get_charsets(self): charsets = self._charsets if charsets is None: envadapter = IUserPreferredCharsets(self) charsets = envadapter.getPreferredCharsets() or ['utf-8'] self._charsets = charsets return charsets
def processInputs(request, charsets=None): """Process the values in request.form to decode strings to unicode, using the passed-in list of charsets. If none are passed in, look up the user's preferred charsets. The default is to use utf-8. """ if IProcessedRequest.providedBy(request): return if charsets is None: envadapter = IUserPreferredCharsets(request, None) if envadapter is None: charsets = ['utf-8'] else: charsets = envadapter.getPreferredCharsets() or ['utf-8'] for name, value in request.form.items(): if not (isCGI_NAME(name) or name.startswith('HTTP_')): if isinstance(value, str): request.form[name] = _decode(value, charsets) elif isinstance(value, (list, tuple,)): newValue = [] for val in value: if isinstance(val, str): val = _decode(val, charsets) newValue.append(val) if isinstance(value, tuple): newValue = tuple(value) request.form[name] = newValue interface.alsoProvides(request, IProcessedRequest)
def test_adapter_registered(self): """ test naaya.i18n's IPreferredCharset adapter is registered """ req = create_request() adapter = IUserPreferredCharsets(req) self.assertTrue(isinstance(adapter, NyHTTPCharsets)) self.assertEqual(adapter.getPreferredCharsets(), ['utf-8', 'iso-8859-1', '*'])
def _setPageEncoding(self): """Set the encoding of the form page via the Content-Type header. ZPublisher uses the value of this header to determine how to encode unicode data for the browser.""" envadapter = IUserPreferredCharsets(self.request) charsets = envadapter.getPreferredCharsets() or ['utf-8'] self.request.RESPONSE.setHeader( 'Content-Type', 'text/html; charset=%s' % charsets[0])
def _setPageEncoding(self): """Set the encoding of the form page via the Content-Type header. ZPublisher uses the value of this header to determine how to encode unicode data for the browser.""" envadapter = IUserPreferredCharsets(self.request) charsets = envadapter.getPreferredCharsets() or ['utf-8'] self.request.RESPONSE.setHeader('Content-Type', 'text/html; charset=%s' % charsets[0])
def processInputs(request, charsets=None): if charsets is None: envadapter = IUserPreferredCharsets(request) charsets = envadapter.getPreferredCharsets() or ['utf-8'] for name, value in request.form.items(): if (not (isCGI_NAME(name) or name.startswith('HTTP_')) and isinstance(value, str)): request.form[name] = _decode(value, charsets)
def __call__(self): # XXX dirty hack to make the values coming out of here # encoded properly, by default please fix me. envadapter = IUserPreferredCharsets(self.request) charsets = envadapter.getPreferredCharsets() or ['utf-8'] value = self.template() if not isinstance(value, unicode): value = decode._decode(self.template(), charsets) return value
def _decode(self, text): """Try to decode the text using one of the available charsets.""" if self.charsets is None: envadapter = IUserPreferredCharsets(self.request) self.charsets = envadapter.getPreferredCharsets() or ['utf-8'] for charset in self.charsets: try: text = unicode(text, charset) break except UnicodeError: pass return text
def _decode(self, text): """Try to decode the text using one of the available charsets.""" if self.charsets is None: envadapter = IUserPreferredCharsets(self.request) self.charsets = envadapter.getPreferredCharsets() or ['utf-8'] for charset in self.charsets: try: text = unicode(text, charset) break except UnicodeError: pass return text
def setPageEncoding(request): """Set the encoding of the form page via the Content-Type header. ZPublisher uses the value of this header to determine how to encode unicode data for the browser. """ warn(u'setPageEncoding() is deprecated and will be removed in Zope 2.16. ' u'It is recommended to let the ZPublisher use the default_encoding. ' u'Please consider setting default-zpublisher-encoding to utf-8.', DeprecationWarning, stacklevel=2) envadapter = IUserPreferredCharsets(request) charsets = envadapter.getPreferredCharsets() or ['utf-8'] request.RESPONSE.setHeader( 'Content-Type', 'text/html; charset=%s' % charsets[0])
def setPageEncoding(request): """Set the encoding of the form page via the Content-Type header. ZPublisher uses the value of this header to determine how to encode unicode data for the browser. """ warn(u'setPageEncoding() is deprecated and will be removed in Zope 5.0. ' u'It is recommended to let the ZPublisher use the default_encoding. ' u'Please consider setting default-zpublisher-encoding to utf-8.', DeprecationWarning, stacklevel=2) envadapter = IUserPreferredCharsets(request) charsets = envadapter.getPreferredCharsets() or ['utf-8'] request.RESPONSE.setHeader( 'Content-Type', 'text/html; charset=%s' % charsets[0])
def getCharsetUsingRequest(request): 'See IHTTPResponse' envadapter = IUserPreferredCharsets(request, None) if envadapter is None: return try: charset = envadapter.getPreferredCharsets()[0] except IndexError: # Exception caused by empty list! This is okay though, since the # browser just could have sent a '*', which means we can choose # the encoding, which we do here now. charset = 'utf-8' return charset
def getCharsetUsingRequest(request): 'See IHTTPResponse' envadapter = IUserPreferredCharsets(request, None) if envadapter is None: return try: charset = envadapter.getPreferredCharsets()[0] except IndexError: # Exception caused by empty list! This is okay though, since the # browser just could have sent a '*', which means we can choose # the encoding, which we do here now. charset = 'utf-8' return charset
def processInputs(request, charsets=None): """Process the values in request.form to decode strings to unicode, using the passed-in list of charsets. If none are passed in, look up the user's preferred charsets. The default is to use utf-8. """ if charsets is None: envadapter = IUserPreferredCharsets(request, None) if envadapter is None: charsets = ['utf-8'] else: charsets = envadapter.getPreferredCharsets() or ['utf-8'] for name, value in request.form.items(): if not (name in isCGI_NAMEs or name.startswith('HTTP_')): request.form[name] = processInputValue(value, charsets)
def processInputs(request, charsets=None): if charsets is None: envadapter = IUserPreferredCharsets(request) charsets = envadapter.getPreferredCharsets() or ['utf-8'] for name, value in request.form.items(): if not (isCGI_NAME(name) or name.startswith('HTTP_')): if isinstance(value, str): request.form[name] = _decode(value, charsets) elif isinstance(value, list): request.form[name] = [ _decode(val, charsets) for val in value if isinstance(val, str) ] elif isinstance(value, tuple): request.form[name] = tuple([ _decode(val, charsets) for val in value if isinstance(val, str) ])
def to_unicode(text): if not charsets: envadapter = IUserPreferredCharsets(request, None) if envadapter: pref = envadapter.getPreferredCharsets() if pref: charsets.extend(pref) if not charsets: charsets.append('utf-8') for charset in charsets: try: return unicode(text, charset) except UnicodeError: pass raise UnicodeError( "Unable to decode %s using any of the character sets: %s" % (repr(text), repr(charsets)))
def processInputs(request, charsets=None): if charsets is None: envadapter = IUserPreferredCharsets(request) charsets = envadapter.getPreferredCharsets() or ['utf-8'] for name, value in request.form.items(): if not (isCGI_NAME(name) or name.startswith('HTTP_')): if isinstance(value, str): request.form[name] = _decode(value, charsets) elif isinstance(value, list): request.form[name] = [ _decode(val, charsets) for val in value if isinstance(val, str) ] elif isinstance(value, tuple): request.form[name] = tuple([ _decode(val, charsets) for val in value if isinstance(val, str) ])
def _decode(self, text): """Try to decode the text using one of the available charsets.""" # According to PEP-3333, in python-3, QUERY_STRING is a string, # representing 'latin-1' encoded byte array. So, if we are in python-3 # context, encode text as 'latin-1' first, to try to decode # resulting byte array using user-supplied charset. if not isinstance(text, bytes): text = text.encode('latin-1') if self.charsets is None: envadapter = IUserPreferredCharsets(self) self.charsets = envadapter.getPreferredCharsets() or ['utf-8'] self.charsets = [c for c in self.charsets if c != '*'] for charset in self.charsets: try: text = _u(text, charset) break except UnicodeError: pass return text
def processInputs(request, charsets=None): """Process the values in request.form to decode strings to unicode, using the passed-in list of charsets. If none are passed in, look up the user's preferred charsets. The default is to use utf-8. """ warn(u'processInputs() is deprecated and will be removed in Zope 2.16. If ' u'your view implements IBrowserPage, similar processing is now ' u'executed automatically.', DeprecationWarning, stacklevel=2) if charsets is None: envadapter = IUserPreferredCharsets(request, None) if envadapter is None: charsets = ['utf-8'] else: charsets = envadapter.getPreferredCharsets() or ['utf-8'] for name, value in request.form.items(): if not (name in isCGI_NAMEs or name.startswith('HTTP_')): request.form[name] = processInputValue(value, charsets)
def processInputs(request, charsets=None): """Process the values in request.form to decode binary_type to text_type, using the passed-in list of charsets. If none are passed in, look up the user's preferred charsets. The default is to use utf-8. """ warn(u'processInputs() is deprecated and will be removed in Zope 5.0. If ' u'your view implements IBrowserPage, similar processing is now ' u'executed automatically.', DeprecationWarning, stacklevel=2) if charsets is None: envadapter = IUserPreferredCharsets(request, None) if envadapter is None: charsets = ['utf-8'] else: charsets = envadapter.getPreferredCharsets() or ['utf-8'] for name, value in list(request.form.items()): if not (name in isCGI_NAMEs or name.startswith('HTTP_')): request.form[name] = processInputValue(value, charsets)
def resolve(self, context, text, expression): request = context.REQUEST charsets = getattr(request, '__zpt_available_charsets', None) if charsets is None: charsets = IUserPreferredCharsets(request).getPreferredCharsets() # add management_page_charset as one fallback management_charset = getattr(context, 'management_page_charset', None) if management_charset: charsets.append(management_charset) # add Python's default encoding as last fallback charsets.append(default_encoding) # cache list of charsets request.__zpt_available_charsets = charsets for enc in charsets: if enc == '*': continue try: return unicode(text, enc) except (LookupError, UnicodeDecodeError): pass return text
def processInputs(request, charsets=None): """Process the values in request.form to decode strings to unicode, using the passed-in list of charsets. If none are passed in, look up the user's preferred charsets. The default is to use utf-8. """ warn( u"processInputs() is deprecated and will be removed in Zope 2.16. If " u"your view implements IBrowserPage, similar processing is now " u"executed automatically.", DeprecationWarning, stacklevel=2, ) if charsets is None: envadapter = IUserPreferredCharsets(request, None) if envadapter is None: charsets = ["utf-8"] else: charsets = envadapter.getPreferredCharsets() or ["utf-8"] for name, value in request.form.items(): if not (name in isCGI_NAMEs or name.startswith("HTTP_")): request.form[name] = processInputValue(value, charsets)
def resolve(self, context, text, expression): if isinstance(text, text_type): return text request = aq_get(context, 'REQUEST', None) # Deal with the fact that a REQUEST is not always available. # In this case fall back to the encoding of the ZMI and the # Python default encoding. if request is None: charsets = [ ZPublisher.HTTPRequest.default_encoding, default_encoding ] else: # charsets might by cached within the request charsets = getattr(request, '__zpt_available_charsets', None) # No uncached charsets found: investigate the HTTP_ACCEPT_CHARSET # header. This code is only called if 'context' has a request # object. The condition is true because otherwise 'charsets' contains # at least the default encoding of Python. if charsets is None: charsets = list() # add Python's default encoding as last fallback charsets.append(default_encoding) # include the charsets based on the HTTP_ACCEPT_CHARSET # header charsets = IUserPreferredCharsets( request).getPreferredCharsets() + charsets # cache list of charsets request.__zpt_available_charsets = charsets for enc in charsets: if enc == '*': continue try: return text.decode(enc) except (LookupError, UnicodeDecodeError): pass # FIXME: Shouldn't this raise an Exception or signal an error somehow? return text
def resolve(self, context, text, expression): request = aq_get(context, 'REQUEST', None) # Deal with the fact that a REQUEST is not always available. # In this case fall back to the encoding of the ZMI and the # Python default encoding. if request is None: charsets = [default_encoding] management_charset = getattr(context, 'management_page_charset', None) if management_charset: charsets.insert(0, management_charset) else: # charsets might by cached within the request charsets = getattr(request, '__zpt_available_charsets', None) # No uncached charsets found: investigate the HTTP_ACCEPT_CHARSET # header. This code is only called if 'context' has a request # object. The condition is true because otherwise 'charsets' contains # at least the default encoding of Python. if charsets is None: charsets = list() # add Python's default encoding as last fallback charsets.append(default_encoding) # include the charsets based on the HTTP_ACCEPT_CHARSET # header charsets = IUserPreferredCharsets(request).getPreferredCharsets() +\ charsets # cache list of charsets request.__zpt_available_charsets = charsets for enc in charsets: if enc == '*': continue try: return unicode(text, enc) except (LookupError, UnicodeDecodeError): pass return text
def getBrowserCharset(request): """ Get charset preferred by the browser. """ envadapter = IUserPreferredCharsets(request) charsets = envadapter.getPreferredCharsets() or ['utf-8'] return charsets[0]
def getBrowserCharset(request): """ Get charset preferred by the browser. """ envadapter = IUserPreferredCharsets(request) charsets = envadapter.getPreferredCharsets() or ['utf-8'] return charsets[0]