def _formatparam(param, value=None, quote=True): """Convenience function to format and return a key=value pair. This will quote the value if needed or if quote is true. If value is a three tuple (charset, language, value), it will be encoded according to RFC2231 rules. If it contains non-ascii characters it will likewise be encoded according to RFC2231 rules, using the utf-8 charset and a null language. """ if value is not None and len(value) > 0: if isinstance(value, tuple): param += '*' value = utils.encode_rfc2231(value[2], value[0], value[1]) return '%s=%s' % (param, value) else: try: value.encode('ascii') except UnicodeEncodeError: param += '*' value = utils.encode_rfc2231(value, 'utf-8', '') return '%s=%s' % (param, value) if quote or tspecials.search(value): return '%s="%s"' % (param, utils.quote(value)) else: return '%s=%s' % (param, value) else: return param
def _formatparam(param, value=None, quote=True): """Convenience function to format and return a key=value pair. This will quote the value if needed or if quote is true. If value is a three tuple (charset, language, value), it will be encoded according to RFC2231 rules. If it contains non-ascii characters it will likewise be encoded according to RFC2231 rules, using the utf-8 charset and a null language. """ if value is not None and len(value) > 0: # A tuple is used for RFC 2231 encoded parameter values where items # are (charset, language, value). charset is a string, not a Charset # instance. RFC 2231 encoded values are never quoted, per RFC. if isinstance(value, tuple): # Encode as per RFC 2231 param += '*' value = utils.encode_rfc2231(value[2], value[0], value[1]) return '%s=%s' % (param, value) else: try: value.encode('ascii') except UnicodeEncodeError: param += '*' value = utils.encode_rfc2231(value, 'utf-8', '') return '%s=%s' % (param, value) # BAW: Please check this. I think that if quote is set it should # force quoting even if not necessary. if quote or tspecials.search(value): return '%s="%s"' % (param, utils.quote(value)) else: return '%s=%s' % (param, value) else: return param
def json_writer(response, fields, name=None, bom=False): u'''Context manager for writing UTF-8 JSON data to response :param response: file-like or response-like object for writing data and headers (response-like objects only) :param fields: list of datastore fields :param name: file name (for headers, response-like objects only) :param bom: True to include a UTF-8 BOM at the start of the file >>> with json_writer(response, fields) as d: >>> d.writerow(row1) >>> d.writerow(row2) ''' if hasattr(response, u'headers'): response.headers['Content-Type'] = ( b'application/json; charset=utf-8') if name: response.headers['Content-disposition'] = ( b'attachment; filename="{name}.json"'.format( name=encode_rfc2231(name))) if bom: response.write(UTF8_BOM) response.write( b'{\n "fields": %s,\n "records": [' % json.dumps( fields, ensure_ascii=False, separators=(u',', u':'))) yield JSONWriter(response, [f['id'] for f in fields]) response.write(b'\n]}\n')
def json_writer(response: Any, fields: list[dict[str, Any]], name: Optional[str] = None, bom: bool = False): u'''Context manager for writing UTF-8 JSON data to response :param response: file-like or response-like object for writing data and headers (response-like objects only) :param fields: list of datastore fields :param name: file name (for headers, response-like objects only) :param bom: True to include a UTF-8 BOM at the start of the file ''' if hasattr(response, u'headers'): response.headers['Content-Type'] = (b'application/json; charset=utf-8') if name: response.headers['Content-disposition'] = ( u'attachment; filename="{name}.json"'.format( name=encode_rfc2231(name))) if bom: response.stream.write(BOM_UTF8) response.stream.write( six.ensure_binary( u'{\n "fields": %s,\n "records": [' % dumps(fields, ensure_ascii=False, separators=(u',', u':')))) yield JSONWriter(response.stream) response.stream.write(b'\n]}\n')
def tsv_writer(response: Any, fields: list[dict[str, Any]], name: Optional[str] = None, bom: bool = False): u'''Context manager for writing UTF-8 TSV data to response :param response: file-like or response-like object for writing data and headers (response-like objects only) :param fields: list of datastore fields :param name: file name (for headers, response-like objects only) :param bom: True to include a UTF-8 BOM at the start of the file ''' if hasattr(response, u'headers'): response.headers['Content-Type'] = ( b'text/tab-separated-values; charset=utf-8') if name: response.headers['Content-disposition'] = ( u'attachment; filename="{name}.tsv"'.format( name=encode_rfc2231(name))) if bom: response.stream.write(BOM_UTF8) csv.writer(response.stream, dialect='excel-tab').writerow(f['id'] for f in fields) yield TextWriter(response.stream)
def get_request_params(self, api_url): params = super(MailgunPayload, self).get_request_params(api_url) non_ascii_filenames = [ filename for (field, (filename, content, mimetype)) in params["files"] if filename is not None and not isascii(filename) ] if non_ascii_filenames: # Workaround https://github.com/requests/requests/issues/4652: # Mailgun expects RFC 7578 compliant multipart/form-data, and is confused # by Requests/urllib3's improper use of RFC 2231 encoded filename parameters # ("filename*=utf-8''...") in Content-Disposition headers. # The workaround is to pre-generate the (non-compliant) form-data body, and # replace 'filename*={RFC 2231 encoded}' with 'filename="{UTF-8 bytes}"'. # Replace _only_ the filenames that will be problems (not all "filename*=...") # to minimize potential side effects--e.g., in attached messages that might # have their own attachments with (correctly) RFC 2231 encoded filenames. prepared = Request(**params).prepare() form_data = prepared.body # bytes for filename in non_ascii_filenames: # text rfc2231_filename = encode_rfc2231( # wants a str (text in PY3, bytes in PY2) filename if isinstance(filename, str) else filename.encode("utf-8"), charset="utf-8") form_data = form_data.replace( b'filename*=' + rfc2231_filename.encode("utf-8"), b'filename="' + filename.encode("utf-8") + b'"') params["data"] = form_data params["headers"]["Content-Type"] = prepared.headers[ "Content-Type"] # multipart/form-data; boundary=... params["files"] = None # these are now in the form_data body return params
def xml_writer(response, fields, name=None, bom=False): u'''Context manager for writing UTF-8 XML data to response :param response: file-like or response-like object for writing data and headers (response-like objects only) :param fields: list of datastore fields :param name: file name (for headers, response-like objects only) :param bom: True to include a UTF-8 BOM at the start of the file >>> with xml_writer(response, fields) as d: >>> d.writerow(row1) >>> d.writerow(row2) ''' if hasattr(response, u'headers'): response.headers['Content-Type'] = ( b'text/xml; charset=utf-8') if name: response.headers['Content-disposition'] = ( b'attachment; filename="{name}.xml"'.format( name=encode_rfc2231(name))) if bom: response.write(UTF8_BOM) response.write(b'<data>\n') yield XMLWriter(response, [f['id'] for f in fields]) response.write(b'</data>\n')
def encode_params(value): """RFC 5987(8187) (specialized from RFC 2231) parameter encoding. This encodes parameters as specified by RFC 5987 using fixed `UTF-8` encoding (as required by RFC 8187). However, all parameters with non latin-1 values are automatically transformed and a `*` suffixed parameter is added (RFC 8187 allows this only for parameters explicitly specified to have this behavior). Many HTTP headers use `,` separated lists. For simplicity, such headers are not supported (we would need to recognize `,` inside quoted strings as special). """ params = [] for p in _parseparam(";" + value): p = p.strip() if not p: continue params.append([s.strip() for s in p.split("=", 1)]) known_params = {p[0] for p in params} for p in params[:]: if len(p) == 2 and non_latin_1(p[1]): # need encoding pn = p[0] pnc = pn + "*" pv = p[1] if pnc not in known_params: if pv.startswith('"'): pv = pv[1:-1] # remove quotes params.append((pnc, encode_rfc2231(pv, "utf-8", None))) # backward compatibility for clients not understanding RFC 5987 p[1] = p[1].encode("iso-8859-1", "replace").decode("iso-8859-1") return "; ".join("=".join(p) for p in params)
def tsv_writer(response, fields, name=None, bom=False): u'''Context manager for writing UTF-8 TSV data to response :param response: file-like or response-like object for writing data and headers (response-like objects only) :param fields: list of datastore fields :param name: file name (for headers, response-like objects only) :param bom: True to include a UTF-8 BOM at the start of the file >>> with tsv_writer(response, fields) as d: >>> d.writerow(row1) >>> d.writerow(row2) ''' if hasattr(response, u'headers'): response.headers['Content-Type'] = ( b'text/tab-separated-values; charset=utf-8') if name: response.headers['Content-disposition'] = ( b'attachment; filename="{name}.tsv"'.format( name=encode_rfc2231(name))) wr = CSVWriter( response, fields, encoding=u'utf-8', dialect=unicodecsv.excel_tab, ) if bom: response.write(UTF8_BOM) wr.writerow(f['id'] for f in fields) yield wr
def tsv_writer(response, fields, name=None, bom=False): u'''Context manager for writing UTF-8 TSV data to response :param response: file-like or response-like object for writing data and headers (response-like objects only) :param fields: list of datastore fields :param name: file name (for headers, response-like objects only) :param bom: True to include a UTF-8 BOM at the start of the file >>> with tsv_writer(response, fields) as d: >>> d.writerow(row1) >>> d.writerow(row2) ''' if hasattr(response, u'headers'): response.headers['Content-Type'] = ( b'text/tab-separated-values; charset=utf-8') if name: response.headers['Content-disposition'] = ( b'attachment; filename="{name}.tsv"'.format( name=encode_rfc2231(name))) wr = unicodecsv.writer( response, encoding=u'utf-8', dialect=unicodecsv.excel_tab) if bom: response.write(UTF8_BOM) wr.writerow(f['id'] for f in fields) yield wr
def _formatparam(param, value=None, quote=True): if value is not None and len(value) > 0: if isinstance(value, tuple): param += '*' value = utils.encode_rfc2231(value[2], value[0], value[1]) return '%s=%s' % (param, value) try: value.encode('ascii') except UnicodeEncodeError: param += '*' value = utils.encode_rfc2231(value, 'utf-8', '') return '%s=%s' % (param, value) if quote or tspecials.search(value): return '%s="%s"' % (param, utils.quote(value)) return '%s=%s' % (param, value) else: return param
def encode_header_value(value: Union[str, datetime, Number], params: Dict[str, Union[str, datetime, Number]], *, encoding: str = 'ascii', always_quote: bool = False) -> bytes: """ Encode a structured header value for transmission over the network. If a parameter value cannot be encoded to the given encoding, the :rfc:`5987` method is used to encode the value in an alternate field using :rfc:`2231` encoding where the field name ends with ``*``. The original field will have an urlencoded version of the value. Any datetimes in the ``value`` argument or any of the parameter values will be formatted as defined by :rfc:`822`. Any numeric values will be converted to strings. If a parameter value is ``False`` or ``None``, the parameter is omitted entirely from the output. If the value is ``True``, only the key will be included in the output (without any ``=``). All other value types are disallowed. :param value: the main value of the header :param params: a dictionary of parameter names and values :param encoding: the character encoding to use (either ``ascii`` or ``iso-8859-1``) :param always_quote: always enclose the parameter values in quotes, even if it's unnecessary :return: the encoded bytestring """ def transform_value(val): if isinstance(val, str): return val elif isinstance(val, datetime): return format_datetime(val.astimezone(timezone.utc), usegmt=True) else: return str(val) assert check_argument_types() buffer = transform_value(value).encode(encoding) for key, value in params.items(): key = key.encode(encoding) buffer += b'; ' + key value = transform_value(value) quoted_value = quote(value) add_quotes = always_quote or quoted_value != value try: quoted_value = quoted_value.encode(encoding) except UnicodeEncodeError: ascii_value = urllib_quote(quoted_value).encode('ascii') rfc2231_value = encode_rfc2231(quoted_value, 'utf-8').encode('utf-8') if add_quotes: ascii_value = b'"' + ascii_value + b'"' rfc2231_value = b'"' + rfc2231_value + b'"' buffer += b'=' + ascii_value + b'; ' + key + b'*=' + rfc2231_value else: if add_quotes: quoted_value = b'"' + quoted_value + b'"' buffer += b'=' + quoted_value return buffer
def _formatparam(param, value = None, quote = True): if value is not None and len(value) > 0: if isinstance(value, tuple): param += '*' value = utils.encode_rfc2231(value[2], value[0], value[1]) if quote or tspecials.search(value): return '%s="%s"' % (param, utils.quote(value)) else: return '%s=%s' % (param, value) else: return param
def _formatparam(param, value=None, quote=True): if value is not None and len(value) > 0: if isinstance(value, tuple): param += '*' value = utils.encode_rfc2231(value[2], value[0], value[1]) if quote or tspecials.search(value): return '%s="%s"' % (param, utils.quote(value)) else: return '%s=%s' % (param, value) else: return param
def serve_request(request, connection, _id=None): fs = GridFS(connection[settings.MONGO_DB_NAME]) try: file = fs.get(_id) except: abort(404) if getattr(file, 'pending', False): abort(404) if getattr(file, 'deleted', False): abort(404) if request.if_modified_since: if request.if_modified_since > file.uploadDate: return Response(status=304) if request.if_none_match: if request.if_none_match.contains(file.md5): return Response(status=304) filename = file.filename # Process non-latin filenames using technique described here: # http://greenbytes.de/tech/tc2231/#encoding-2231-fb rfc2231_filename = encode_rfc2231(filename.encode('utf-8'), 'UTF-8') transliterated_filename = unidecode(filename) content_disposition = 'inline; filename="%s"; filename*=%s;' % ( transliterated_filename, rfc2231_filename ) headers = {'Content-Disposition': content_disposition} callbacks = [connection.close] if not request.range: return serve_full_file_request(request, headers, file, callbacks=callbacks) if request.range.units != 'bytes': abort(400) ranges = request.range.ranges if len(ranges) > 1: return serve_full_file_request(request, headers, file, callbacks=callbacks) start, end = ranges[0] if end is None: end = file.length elif end > file.length: abort(416) return serve_partial_file_request(request, headers, file, start, end, callbacks=callbacks)
def _formatparam(param, value=None, quote=True): """Convenience function to format and return a key=value pair. This will quote the value if needed or if quote is true. """ if value is not None and len(value) > 0: if isinstance(value, tuple): param += '*' value = utils.encode_rfc2231(value[2], value[0], value[1]) if quote or tspecials.search(value): return '%s="%s"' % (param, utils.quote(value)) else: return '%s=%s' % (param, value) else: return param
def _formatparam(param, value=None, quote=True): """This is _formatparam from Python 2.7""" if value is not None and len(value) > 0: # A tuple is used for RFC 2231 encoded parameter values where items # are (charset, language, value). charset is a string, not a Charset # instance. if isinstance(value, tuple): # Encode as per RFC 2231 param += '*' value = utils.encode_rfc2231(value[2], value[0], value[1]) # BAW: Please check this. I think that if quote is set it should # force quoting even if not necessary. if quote or message.tspecials.search(value): return '%s="%s"' % (param, utils.quote(value)) else: return '%s=%s' % (param, value) else: return param
def _formatparam(param, value = None, quote = True): """Convenience function to format and return a key=value pair. This will quote the value if needed or if quote is true. If value is a three tuple (charset, language, value), it will be encoded according to RFC2231 rules. """ if value is not None and len(value) > 0: if isinstance(value, tuple): param += '*' value = utils.encode_rfc2231(value[2], value[0], value[1]) if quote or tspecials.search(value): return '%s="%s"' % (param, utils.quote(value)) else: return '%s=%s' % (param, value) else: return param return
def mailto_format(**kwargs): # @TODO: implement utf8 option kwargs = _fix_addersses(**kwargs) parts = [] for headername in ('to', 'cc', 'bcc', 'subject', 'body', 'attach'): if headername in kwargs: headervalue = kwargs[headername] if not headervalue: continue if headername in ('address', 'to', 'cc', 'bcc'): parts.append('%s=%s' % (headername, headervalue)) else: headervalue = encode_rfc2231(headervalue) # @TODO: check parts.append('%s=%s' % (headername, headervalue)) mailto_string = 'mailto:%s' % kwargs.get('address', '') if parts: mailto_string = '%s?%s' % (mailto_string, '&'.join(parts)) return mailto_string
def csv_writer(response, fields, name=None, bom=False): u'''Context manager for writing UTF-8 CSV data to response :param response: file-like or response-like object for writing data and headers (response-like objects only) :param fields: list of datastore fields :param name: file name (for headers, response-like objects only) :param bom: True to include a UTF-8 BOM at the start of the file ''' if hasattr(response, u'headers'): response.headers['Content-Type'] = b'text/csv; charset=utf-8' if name: response.headers['Content-disposition'] = ( u'attachment; filename="{name}.csv"'.format( name=encode_rfc2231(name))) if bom: response.stream.write(BOM_UTF8) csv.writer(response.stream).writerow(f['id'] for f in fields) yield TextWriter(response.stream)
def _formatparam(param, value=None, quote=True): """Convenience function to format and return a key=value pair. This will quote the value if needed or if quote is true. """ if value is not None and len(value) > 0: # A tuple is used for RFC 2231 encoded parameter values where items # are (charset, language, value). charset is a string, not a Charset # instance. if isinstance(value, tuple): # Encode as per RFC 2231 param += '*' value = utils.encode_rfc2231(value[2], value[0], value[1]) # BAW: Please check this. I think that if quote is set it should # force quoting even if not necessary. if quote or tspecials.search(value): return '%s="%s"' % (param, utils.quote(value)) else: return '%s=%s' % (param, value) else: return param
def csv_writer(response, fields, name=None, bom=False): u'''Context manager for writing UTF-8 CSV data to response :param response: file-like or response-like object for writing data and headers (response-like objects only) :param fields: list of datastore fields :param name: file name (for headers, response-like objects only) :param bom: True to include a UTF-8 BOM at the start of the file ''' if hasattr(response, u'headers'): response.headers['Content-Type'] = b'text/csv; charset=utf-8' if name: response.headers['Content-disposition'] = ( b'attachment; filename="{name}.csv"'.format( name=encode_rfc2231(name))) if bom: response.write(BOM_UTF8) unicodecsv.writer(response, encoding=u'utf-8').writerow( f['id'] for f in fields) yield TextWriter(response)
def xml_writer(response: Any, fields: list[dict[str, Any]], name: Optional[str] = None, bom: bool = False): u'''Context manager for writing UTF-8 XML data to response :param response: file-like or response-like object for writing data and headers (response-like objects only) :param fields: list of datastore fields :param name: file name (for headers, response-like objects only) :param bom: True to include a UTF-8 BOM at the start of the file ''' if hasattr(response, u'headers'): response.headers['Content-Type'] = (b'text/xml; charset=utf-8') if name: response.headers['Content-disposition'] = ( u'attachment; filename="{name}.xml"'.format( name=encode_rfc2231(name))) if bom: response.stream.write(BOM_UTF8) response.stream.write(b'<data>\n') yield XMLWriter(response.stream, [f[u'id'] for f in fields]) response.stream.write(b'</data>\n')
def serve_request(request, connection, _id=None, filename=None): fs = GridFS(connection[settings.MONGO_DB_NAME]) if _id: try: file = fs.get(_id) except: abort(404) elif filename: try: file = fs.get_last_version(filename) except: abort(404) else: abort(404) if getattr(file, 'pending', False): abort(404) if getattr(file, 'deleted', False): abort(404) if request.if_modified_since: if request.if_modified_since > file.uploadDate: return Response(status=304) if request.if_none_match: if request.if_none_match.contains(file.md5): return Response(status=304) filename = file.filename # Process non-latin filenames using technique described here: # http://greenbytes.de/tech/tc2231/#encoding-2231-fb rfc2231_filename = encode_rfc2231(filename.encode('utf-8'), 'UTF-8') transliterated_filename = unidecode(filename) content_disposition = 'inline; filename="%s"; filename*=%s;' % ( transliterated_filename, rfc2231_filename) headers = {'Content-Disposition': content_disposition} callbacks = [connection.close] if not request.range: return serve_full_file_request(request, headers, file, callbacks=callbacks) if request.range.units != 'bytes': abort(400) ranges = request.range.ranges if len(ranges) > 1: return serve_full_file_request(request, headers, file, callbacks=callbacks) start, end = ranges[0] if end is None: end = file.length elif end > file.length: abort(416) return serve_partial_file_request(request, headers, file, start, end, callbacks=callbacks)