def render(self, data, accepted_media_type=None, renderer_context=None): """ Render `obj` into json. """ if data is None: return '' # If 'indent' is provided in the context, then pretty print the result. # E.g. If we're being called by the BrowsableAPIRenderer. renderer_context = renderer_context or {} indent = renderer_context.get('indent', None) if accepted_media_type: # If the media type looks like 'application/json; indent=4', # then pretty print the result. base_media_type, params = parse_header(accepted_media_type.encode('ascii')) indent = params.get('indent', indent) try: indent = max(min(int(indent), 8), 0) except (ValueError, TypeError): indent = None ret = json.dumps(data, cls=self.encoder_class, indent=indent, ensure_ascii=self.ensure_ascii) if not self.ensure_ascii: return bytes(ret.encode(self.charset)) return ret
def render(self, data, accepted_media_type=None, renderer_context=None): """ Render `data` into JSON. """ if data is None: return "" # If 'indent' is provided in the context, then pretty print the result. # E.g. If we're being called by the BrowsableAPIRenderer. renderer_context = renderer_context or {} indent = renderer_context.get("indent", None) if accepted_media_type: # If the media type looks like 'application/json; indent=4', # then pretty print the result. base_media_type, params = parse_header(accepted_media_type.encode("ascii")) indent = params.get("indent", indent) try: indent = max(min(int(indent), 8), 0) except (ValueError, TypeError): indent = None ret = json.dumps(data, cls=self.encoder_class, indent=indent, ensure_ascii=self.ensure_ascii) # On python 2.x json.dumps() returns bytestrings if ensure_ascii=True, # but if ensure_ascii=False, the return type is underspecified, # and may (or may not) be unicode. # On python 3.x json.dumps() returns unicode strings. if isinstance(ret, six.text_type): return bytes(ret.encode(self.charset)) return ret
def is_form_media_type(media_type): """ Return True if the media type is a valid form media type. """ base_media_type, params = parse_header(media_type.encode(HTTP_HEADER_ENCODING)) return (base_media_type == 'application/x-www-form-urlencoded' or base_media_type == 'multipart/form-data')
def __init__(self, media_type_str): if media_type_str is None: media_type_str = '' if not isinstance(media_type_str, bytes): media_type_str = media_type_str.encode(settings.DEFAULT_CHARSET) self.orig = media_type_str self.full_type, self.params = parse_header(media_type_str) self.main_type, sep, self.sub_type = self.full_type.partition('/')
def test_rfc2231_parsing(self): test_data = ( (b"Content-Type: application/x-stuff; title*=us-ascii'en-us'This%20is%20%2A%2A%2Afun%2A%2A%2A", "This is ***fun***"), (b"Content-Type: application/x-stuff; title*=UTF-8''foo-%c3%a4.html", "foo-ä.html"), (b"Content-Type: application/x-stuff; title*=iso-8859-1''foo-%E4.html", "foo-ä.html"), ) for raw_line, expected_title in test_data: parsed = parse_header(raw_line) self.assertEqual(parsed[1]['title'], expected_title)
def get_indent(self, accepted_media_type, renderer_context): if accepted_media_type: # If the media type looks like 'application/json; indent=4', # then pretty print the result. base_media_type, params = parse_header(accepted_media_type.encode('ascii')) try: return max(min(int(params['indent']), 8), 0) except (KeyError, ValueError, TypeError): pass # If 'indent' is provided in the context, then pretty print the result. # E.g. If we're being called by the BrowsableAPIRenderer. return renderer_context.get('indent', None)
def get_indent(self, accepted_media_type, renderer_context): if accepted_media_type: # If the media type looks like 'application/json; indent=4', # then pretty print the result. # Note that we coerce `indent=0` into `indent=None`. base_media_type, params = parse_header(accepted_media_type.encode("ascii")) try: return zero_as_none(max(min(int(params["indent"]), 8), 0)) except (KeyError, ValueError, TypeError): pass # If 'indent' is provided in the context, then pretty print the result. # E.g. If we're being called by the BrowsableAPIRenderer. return renderer_context.get("indent", None)
def get_indent(self, accepted_media_type, renderer_context): if accepted_media_type: # If the media type looks like 'application/json; indent=4', # then pretty print the result. # Note that we coerce `indent=0` into `indent=None`. base_media_type, params = parse_header( accepted_media_type.encode('ascii')) try: return zero_as_none(max(min(int(params['indent']), 8), 0)) except (KeyError, ValueError, TypeError): pass # If 'indent' is provided in the context, then pretty print the result. # E.g. If we're being called by the BrowsableAPIRenderer. return renderer_context.get('indent', None)
def get_filename(self, stream, media_type, parser_context): """ Detects the uploaded file name. First searches a 'filename' url kwarg. Then tries to parse Content-Disposition header. """ try: return parser_context['kwargs']['filename'] except KeyError: pass try: meta = parser_context['request'].META disposition = parse_header(meta['HTTP_CONTENT_DISPOSITION'].encode('utf-8')) return force_text(disposition[1]['filename']) except (AttributeError, KeyError): pass
def get_filename(self, stream, media_type, parser_context): """ Detects the uploaded file name. First searches a 'filename' url kwarg. Then tries to parse Content-Disposition header. """ try: return parser_context['kwargs']['filename'] except KeyError: pass try: meta = parser_context['request'].META disposition = parse_header(meta['HTTP_CONTENT_DISPOSITION'].encode('utf-8')) return force_text(disposition[1]['filename']) except (AttributeError, KeyError): pass
def get_filename(self, stream, media_type, parser_context): """ Detects the uploaded file name. First searches a "filename" url kwarg. Then tries to parse Content-Disposition header. """ try: return parser_context["kwargs"]["filename"] except KeyError: pass try: meta = parser_context["request"].META disposition = parse_header(meta["HTTP_CONTENT_DISPOSITION"]) return disposition[1]["filename"] except (AttributeError, KeyError): pass
def test_rfc2231_wrong_title(self): """ Test wrongly formatted RFC 2231 headers (missing double single quotes). Parsing should not crash (#24209). """ test_data = ( (b"Content-Type: application/x-stuff; title*='This%20is%20%2A%2A%2Afun%2A%2A%2A", b"'This%20is%20%2A%2A%2Afun%2A%2A%2A"), (b"Content-Type: application/x-stuff; title*='foo.html", b"'foo.html"), (b"Content-Type: application/x-stuff; title*=bar.html", b"bar.html"), ) for raw_line, expected_title in test_data: parsed = parse_header(raw_line) self.assertEqual(parsed[1]['title'], expected_title)
def render(self, data, accepted_media_type=None, renderer_context=None): renderer_context = renderer_context or {} indent = renderer_context.get("indent", None) # Pass to the superclass if the Accept header is set with an explicit # indent, if an indent level is manually passed, or if you're attempting # to render `None`. if accepted_media_type: base_media_type, params = parse_header(accepted_media_type.encode("ascii")) indent = params.get("indent", indent) if data is None or indent: return super(SuccinctJSONRenderer, self).render(data, accepted_media_type, renderer_context) return json.dumps( data, cls=self.encoder_class, indent=indent, ensure_ascii=self.ensure_ascii, separators=(",", ":") )
def get_filename(self, stream, media_type, parser_context): """ Detects the uploaded file name. First searches a "filename" url kwarg. Then tries to parse Content-Disposition header. """ try: return parser_context["kwargs"]["filename"] except KeyError: pass try: meta = parser_context["request"].META disposition = parse_header(meta["HTTP_CONTENT_DISPOSITION"]) return disposition[1]["filename"] except (AttributeError, KeyError): pass
def test_rfc2231_wrong_title(self): """ Test wrongly formatted RFC 2231 headers (missing double single quotes). Parsing should not crash (#24209). """ test_data = ( (b"Content-Type: application/x-stuff; title*='This%20is%20%2A%2A%2Afun%2A%2A%2A", b"'This%20is%20%2A%2A%2Afun%2A%2A%2A"), (b"Content-Type: application/x-stuff; title*='foo.html", b"'foo.html"), (b"Content-Type: application/x-stuff; title*=bar.html", b"bar.html"), ) for raw_line, expected_title in test_data: parsed = parse_header(raw_line) self.assertEqual(parsed[1]['title'], expected_title)
def alter_old_distutils_request(request: WSGIRequest): """Alter the request body for compatibility with older distutils clients Due to a bug in the Python distutils library, the request post is sent using \n as a separator instead of the \r\n that the HTTP spec demands. This breaks the Django form parser and therefore we have to write a custom parser. This bug was fixed in the Python 2.7.4 and 3.4: http://bugs.python.org/issue10510 """ # We first need to retrieve the body before accessing POST or FILES since # it can only be read once. body = request.body if request.POST or request.FILES: return new_body = BytesIO() # Split the response in the various parts based on the boundary string content_type, opts = parse_header( request.META['CONTENT_TYPE'].encode('ascii')) parts = body.split(b'\n--' + opts['boundary'] + b'\n') for part in parts: if b'\n\n' not in part: continue headers, content = part.split(b'\n\n', 1) if not headers: continue new_body.write(b'--' + opts['boundary'] + b'\r\n') new_body.write(headers.replace(b'\n', b'\r\n')) new_body.write(b'\r\n\r\n') new_body.write(content) new_body.write(b'\r\n') new_body.write(b'--' + opts['boundary'] + b'--\r\n') request._body = new_body.getvalue() request.META['CONTENT_LENGTH'] = len(request._body) # Clear out _files and _post so that the request object re-parses the body if hasattr(request, '_files'): delattr(request, '_files') if hasattr(request, '_post'): delattr(request, '_post')
def _get_indent(self, accepted_media_type, renderer_context): # If "indent" is provided in the context, then pretty print the result. # E.g. If we"re being called by the BrowsableAPIRenderer. renderer_context = renderer_context or {} indent = renderer_context.get("indent", None) if accepted_media_type: # If the media type looks like "application/json; indent=4", # then pretty print the result. base_media_type, params = parse_header(accepted_media_type.encode("ascii")) indent = params.get("indent", indent) try: indent = max(min(int(indent), 8), 0) except (ValueError, TypeError): indent = None return indent
def _get_indent(self, accepted_media_type, renderer_context): # If "indent" is provided in the context, then pretty print the result. # E.g. If we"re being called by the BrowsableAPIRenderer. renderer_context = renderer_context or {} indent = renderer_context.get("indent", None) if accepted_media_type: # If the media type looks like "application/json; indent=4", # then pretty print the result. base_media_type, params = parse_header(accepted_media_type.encode("ascii")) indent = params.get("indent", indent) try: indent = max(min(int(indent), 8), 0) except (ValueError, TypeError): indent = None return indent
def alter_old_distutils_request(request: WSGIRequest): """Alter the request body for compatibility with older distutils clients Due to a bug in the Python distutils library, the request post is sent using \n as a separator instead of the \r\n that the HTTP spec demands. This breaks the Django form parser and therefore we have to write a custom parser. This bug was fixed in the Python 2.7.4 and 3.4: http://bugs.python.org/issue10510 """ # We first need to retrieve the body before accessing POST or FILES since # it can only be read once. body = request.body if request.POST or request.FILES: return new_body = BytesIO() # Split the response in the various parts based on the boundary string content_type, opts = parse_header(request.META['CONTENT_TYPE'].encode('ascii')) parts = body.split(b'\n--' + opts['boundary'] + b'\n') for part in parts: if b'\n\n' not in part: continue headers, content = part.split(b'\n\n', 1) if not headers: continue new_body.write(b'--' + opts['boundary'] + b'\r\n') new_body.write(headers.replace(b'\n', b'\r\n')) new_body.write(b'\r\n\r\n') new_body.write(content) new_body.write(b'\r\n') new_body.write(b'--' + opts['boundary'] + b'--\r\n') request._body = new_body.getvalue() request.META['CONTENT_LENGTH'] = len(request._body) # Clear out _files and _post so that the request object re-parses the body if hasattr(request, '_files'): delattr(request, '_files') if hasattr(request, '_post'): delattr(request, '_post')
def render(self, data, accepted_media_type=None, renderer_context=None): renderer_context = renderer_context or {} indent = renderer_context.get('indent', None) # Pass to the superclass if the Accept header is set with an explicit # indent, if an indent level is manually passed, or if you're # attempting to render `None`. if accepted_media_type: base_media_type, params = parse_header( accepted_media_type.encode('ascii')) indent = params.get('indent', indent) if data is None or indent: return super(SuccinctJSONRenderer, self).render( data, accepted_media_type, renderer_context) return json.dumps( data, cls=self.encoder_class, indent=indent, ensure_ascii=self.ensure_ascii, separators=(',', ':'))
def test_rfc2231_parsing(self): test_data = ( ( b"Content-Type: application/x-stuff; " b"title*=us-ascii'en-us'This%20is%20%2A%2A%2Afun%2A%2A%2A", "This is ***fun***", ), ( b"Content-Type: application/x-stuff; title*=UTF-8''foo-%c3%a4.html", "foo-ä.html", ), ( b"Content-Type: application/x-stuff; title*=iso-8859-1''foo-%E4.html", "foo-ä.html", ), ) for raw_line, expected_title in test_data: parsed = parse_header(raw_line) self.assertEqual(parsed[1]["title"], expected_title)
def render(self, data, accepted_media_type=None, renderer_context=None): """ Render `obj` into json. """ if data is None: return '' indent = renderer_context.get('indent', None) if accepted_media_type: # If the media type looks like 'application/json; indent=4', # then pretty print the result. base_media_type, params = parse_header(accepted_media_type) indent = params.get('indent', indent) try: indent = max(min(int(indent), 8), 0) except (ValueError, TypeError): indent = None return json.dumps(data, cls=self.encoder_class, indent=indent)
def render(self, data, accepted_media_type=None, renderer_context=None): """ Render `obj` into json. """ if data is None: return '' indent = renderer_context.get('indent', None) if accepted_media_type: # If the media type looks like 'application/json; indent=4', # then pretty print the result. base_media_type, params = parse_header(accepted_media_type) indent = params.get('indent', indent) try: indent = max(min(int(indent), 8), 0) except (ValueError, TypeError): indent = None return json.dumps(data, cls=self.encoder_class, indent=indent)
def get_filename(self, stream, media_type, parser_context): """ Detects the uploaded file name. First searches a 'filename' url kwarg. Then tries to parse Content-Disposition header. """ try: return parser_context["kwargs"]["filename"] except KeyError: pass try: meta = parser_context["request"].META disposition = parse_header( meta["HTTP_CONTENT_DISPOSITION"].encode()) filename_parm = disposition[1] if "filename*" in filename_parm: return self.get_encoded_filename(filename_parm) return force_str(filename_parm["filename"]) except (AttributeError, KeyError, ValueError): pass
def render(self, data, accepted_media_type=None, renderer_context=None): ''' Render `data` into JSON, using the faster ujson library for serialization unless indentation is requested (ujson does not support indentation). ''' if data is None: return bytes() # If 'indent' is provided in the context, then pretty print the result. # E.g. If we're being called by the BrowsableAPIRenderer. renderer_context = renderer_context or {} indent = renderer_context.get('indent', None) if accepted_media_type: # If the media type looks like 'application/json; indent=4', # then pretty print the result. base_media_type, params = parse_header( accepted_media_type.encode('ascii')) indent = params.get('indent', indent) try: indent = max(min(int(indent), 8), 0) except (ValueError, TypeError): indent = None ret = json.dumps(data, cls=self.encoder_class, indent=indent, ensure_ascii=self.ensure_ascii) else: ret = ujson.dumps(data, ensure_ascii=self.ensure_ascii) # On python 2.x json.dumps() returns bytestrings if ensure_ascii=True, # but if ensure_ascii=False, the return type is underspecified, # and may (or may not) be unicode. # On python 3.x json.dumps() returns unicode strings. if isinstance(ret, six.text_type): return bytes(ret.encode('utf-8')) return ret
def render(self, data, accepted_media_type=None, renderer_context=None): """ Render `obj` into json. """ if data is None: return '' # If 'indent' is provided in the context, then pretty print the result. # E.g. If we're being called by the BrowsableAPIRenderer. renderer_context = renderer_context or {} indent = renderer_context.get('indent', None) if accepted_media_type: # If the media type looks like 'application/json; indent=4', # then pretty print the result. base_media_type, params = parse_header(accepted_media_type.encode('ascii')) indent = params.get('indent', indent) try: indent = max(min(int(indent), 8), 0) except (ValueError, TypeError): indent = None return json.dumps(data, cls=self.encoder_class, indent=indent)
def render(self, data, accepted_media_type=None, renderer_context=None): ''' Render `data` into JSON, using the faster ujson library for serialization unless indentation is requested (ujson does not support indentation). ''' if data is None: return bytes() # If 'indent' is provided in the context, then pretty print the result. # E.g. If we're being called by the BrowsableAPIRenderer. renderer_context = renderer_context or {} indent = renderer_context.get('indent', None) if accepted_media_type: # If the media type looks like 'application/json; indent=4', # then pretty print the result. base_media_type, params = parse_header(accepted_media_type.encode('ascii')) indent = params.get('indent', indent) try: indent = max(min(int(indent), 8), 0) except (ValueError, TypeError): indent = None ret = json.dumps(data, cls=self.encoder_class, indent=indent, ensure_ascii=self.ensure_ascii) else: ret = ujson.dumps(data, ensure_ascii=self.ensure_ascii) # On python 2.x json.dumps() returns bytestrings if ensure_ascii=True, # but if ensure_ascii=False, the return type is underspecified, # and may (or may not) be unicode. # On python 3.x json.dumps() returns unicode strings. if isinstance(ret, six.text_type): return bytes(ret.encode('utf-8')) return ret
def __init__(self, media_type_str): if media_type_str is None: media_type_str = '' self.orig = media_type_str self.full_type, self.params = parse_header(media_type_str) self.main_type, sep, self.sub_type = self.full_type.partition('/')
def is_json_request(content_type): """Return the content type is json. """ lhs, rhs = parse_header(content_type) return lhs == 'application/json'
def __init__(self, media_type_str): if media_type_str is None: media_type_str = '' self.orig = media_type_str self.full_type, self.params = parse_header(media_type_str.encode(HTTP_HEADER_ENCODING)) self.main_type, sep, self.sub_type = self.full_type.partition('/')
def parse_header(self, media_type): base_media_type, params = parse_header(media_type) return params.get(self.version_param, self.default_version)
def __init__(self, media_type_str): if media_type_str is None: media_type_str = '' self.orig = media_type_str self.full_type, self.params = parse_header(media_type_str) self.main_type, sep, self.sub_type = self.full_type.partition('/')
def test_parse_header_with_double_quotes_and_semicolon(self): self.assertEqual( parse_header(b'form-data; name="files"; filename="fo\\"o;bar"'), ("form-data", {"name": b"files", "filename": b'fo"o;bar'}), )
def __init__(self, media_type_str): self.orig = "" if (media_type_str is None) else media_type_str self.full_type, self.params = parse_header( self.orig.encode(HTTP_HEADER_ENCODING)) self.main_type, sep, self.sub_type = self.full_type.partition("/")
def fetch_url( url: str, response_delegate: typing.Optional[ typing.Callable[[str, Response], typing.Any] ] = None, user_agent: typing.Optional[str] = None, seen_urls: typing.Optional[typing.List[str]] = None, ) -> typing.Any: """ Fetch a URL, following redirects and denying access to dangerous URLs. If response_delegate is provided, call out to that function to generate the response and return it. Otherwise, just return the file name. """ # delegate fetching of 3rd party service items that have custom URLs, e.g. Google Docs service_item = parse_service_url(url) if service_item: return fetch_service_item(service_item) headers = {"User-Agent": user_agent} if user_agent else None url = convert_raw_content_url(url) url_obj = urlparse(url) if url_obj.hostname is None: raise TypeError('Url "{}" appears to have no hostname.'.format(url)) if is_malicious_host(url_obj.hostname): raise RemoteFileException( "{} is not a valid hostname.".format(url_obj.hostname) ) _, file_ext = splitext(url_obj.path) if seen_urls is None: seen_urls = [] with requests.request( "GET", url, headers=headers, stream=True, timeout=DOWNLOAD_TIMEOUT_SECONDS, allow_redirects=False, ) as resp: if "Location" in resp.headers: # Manually follow redirects to determine if it is a redirect to a malicious URL if url in seen_urls: raise RemoteFileException("Looping redirect found at {}".format(url)) seen_urls.append(url) if len(seen_urls) > MAX_HTTP_REDIRECTS: raise RemoteFileException( "Too many HTTP redirects from original URL: {}".format(seen_urls[0]) ) new_url = urljoin(url, resp.headers["location"]) return fetch_url(new_url, response_delegate, user_agent, seen_urls) resp.raise_for_status() file_name = basename(url_obj.path) if "Content-Disposition" in resp.headers: content_disposition: typing.Union[str, bytes] = resp.headers[ "Content-Disposition" ] if not isinstance(content_disposition, bytes): content_disposition = content_disposition.encode("ascii") _, metadata = parse_header(content_disposition) file_name = metadata.get("filename") if isinstance(file_name, bytes): file_name = file_name.decode("ascii") file_name = unquote(file_name) if response_delegate: return response_delegate(file_name, resp) return file_name
def __init__(self, media_type_str): if media_type_str is None: media_type_str = '' self.orig = media_type_str self.full_type, self.params = parse_header(media_type_str.encode(HTTP_HEADER_ENCODING)) self.main_type, sep, self.sub_type = self.full_type.partition('/')