def render(self, data, accepted_media_type=None, renderer_context=None):
"""
Render `obj` into json.
"""
if data is None:
return ''
# If 'indent' is provided in the context, then pretty print the result.
# E.g. If we're being called by the BrowsableAPIRenderer.
renderer_context = renderer_context or {}
indent = renderer_context.get('indent', None)
if accepted_media_type:
# If the media type looks like 'application/json; indent=4',
# then pretty print the result.
base_media_type, params = parse_header(accepted_media_type.encode('ascii'))
indent = params.get('indent', indent)
try:
indent = max(min(int(indent), 8), 0)
except (ValueError, TypeError):
indent = None
ret = json.dumps(data, cls=self.encoder_class,
indent=indent, ensure_ascii=self.ensure_ascii)
if not self.ensure_ascii:
return bytes(ret.encode(self.charset))
return ret
def render(self, data, accepted_media_type=None, renderer_context=None):
"""
Render `data` into JSON.
"""
if data is None:
return ""
# If 'indent' is provided in the context, then pretty print the result.
# E.g. If we're being called by the BrowsableAPIRenderer.
renderer_context = renderer_context or {}
indent = renderer_context.get("indent", None)
if accepted_media_type:
# If the media type looks like 'application/json; indent=4',
# then pretty print the result.
base_media_type, params = parse_header(accepted_media_type.encode("ascii"))
indent = params.get("indent", indent)
try:
indent = max(min(int(indent), 8), 0)
except (ValueError, TypeError):
indent = None
ret = json.dumps(data, cls=self.encoder_class, indent=indent, ensure_ascii=self.ensure_ascii)
# On python 2.x json.dumps() returns bytestrings if ensure_ascii=True,
# but if ensure_ascii=False, the return type is underspecified,
# and may (or may not) be unicode.
# On python 3.x json.dumps() returns unicode strings.
if isinstance(ret, six.text_type):
return bytes(ret.encode(self.charset))
return ret
def is_form_media_type(media_type):
"""
Return True if the media type is a valid form media type.
"""
base_media_type, params = parse_header(media_type.encode(HTTP_HEADER_ENCODING))
return (base_media_type == 'application/x-www-form-urlencoded' or
base_media_type == 'multipart/form-data')
def __init__(self, media_type_str):
if media_type_str is None:
media_type_str = ''
if not isinstance(media_type_str, bytes):
media_type_str = media_type_str.encode(settings.DEFAULT_CHARSET)
self.orig = media_type_str
self.full_type, self.params = parse_header(media_type_str)
self.main_type, sep, self.sub_type = self.full_type.partition('/')
def test_rfc2231_parsing(self):
test_data = (
(b"Content-Type: application/x-stuff; title*=us-ascii'en-us'This%20is%20%2A%2A%2Afun%2A%2A%2A",
"This is ***fun***"),
(b"Content-Type: application/x-stuff; title*=UTF-8''foo-%c3%a4.html",
"foo-ä.html"),
(b"Content-Type: application/x-stuff; title*=iso-8859-1''foo-%E4.html",
"foo-ä.html"),
)
for raw_line, expected_title in test_data:
parsed = parse_header(raw_line)
self.assertEqual(parsed[1]['title'], expected_title)
def get_indent(self, accepted_media_type, renderer_context):
if accepted_media_type:
# If the media type looks like 'application/json; indent=4',
# then pretty print the result.
base_media_type, params = parse_header(accepted_media_type.encode('ascii'))
try:
return max(min(int(params['indent']), 8), 0)
except (KeyError, ValueError, TypeError):
pass
# If 'indent' is provided in the context, then pretty print the result.
# E.g. If we're being called by the BrowsableAPIRenderer.
return renderer_context.get('indent', None)
def get_indent(self, accepted_media_type, renderer_context):
if accepted_media_type:
# If the media type looks like 'application/json; indent=4',
# then pretty print the result.
# Note that we coerce `indent=0` into `indent=None`.
base_media_type, params = parse_header(accepted_media_type.encode("ascii"))
try:
return zero_as_none(max(min(int(params["indent"]), 8), 0))
except (KeyError, ValueError, TypeError):
pass
# If 'indent' is provided in the context, then pretty print the result.
# E.g. If we're being called by the BrowsableAPIRenderer.
return renderer_context.get("indent", None)
def get_indent(self, accepted_media_type, renderer_context):
if accepted_media_type:
# If the media type looks like 'application/json; indent=4',
# then pretty print the result.
# Note that we coerce `indent=0` into `indent=None`.
base_media_type, params = parse_header(
accepted_media_type.encode('ascii'))
try:
return zero_as_none(max(min(int(params['indent']), 8), 0))
except (KeyError, ValueError, TypeError):
pass
# If 'indent' is provided in the context, then pretty print the result.
# E.g. If we're being called by the BrowsableAPIRenderer.
return renderer_context.get('indent', None)
def get_filename(self, stream, media_type, parser_context):
"""
Detects the uploaded file name. First searches a 'filename' url kwarg.
Then tries to parse Content-Disposition header.
"""
try:
return parser_context['kwargs']['filename']
except KeyError:
pass
try:
meta = parser_context['request'].META
disposition = parse_header(meta['HTTP_CONTENT_DISPOSITION'].encode('utf-8'))
return force_text(disposition[1]['filename'])
except (AttributeError, KeyError):
pass
def get_filename(self, stream, media_type, parser_context):
"""
Detects the uploaded file name. First searches a 'filename' url kwarg.
Then tries to parse Content-Disposition header.
"""
try:
return parser_context['kwargs']['filename']
except KeyError:
pass
try:
meta = parser_context['request'].META
disposition = parse_header(meta['HTTP_CONTENT_DISPOSITION'].encode('utf-8'))
return force_text(disposition[1]['filename'])
except (AttributeError, KeyError):
pass
def get_filename(self, stream, media_type, parser_context):
"""
Detects the uploaded file name. First searches a "filename" url kwarg.
Then tries to parse Content-Disposition header.
"""
try:
return parser_context["kwargs"]["filename"]
except KeyError:
pass
try:
meta = parser_context["request"].META
disposition = parse_header(meta["HTTP_CONTENT_DISPOSITION"])
return disposition[1]["filename"]
except (AttributeError, KeyError):
pass
def test_rfc2231_wrong_title(self):
"""
Test wrongly formatted RFC 2231 headers (missing double single quotes).
Parsing should not crash (#24209).
"""
test_data = (
(b"Content-Type: application/x-stuff; title*='This%20is%20%2A%2A%2Afun%2A%2A%2A",
b"'This%20is%20%2A%2A%2Afun%2A%2A%2A"),
(b"Content-Type: application/x-stuff; title*='foo.html",
b"'foo.html"),
(b"Content-Type: application/x-stuff; title*=bar.html",
b"bar.html"),
)
for raw_line, expected_title in test_data:
parsed = parse_header(raw_line)
self.assertEqual(parsed[1]['title'], expected_title)
def render(self, data, accepted_media_type=None, renderer_context=None):
renderer_context = renderer_context or {}
indent = renderer_context.get("indent", None)
# Pass to the superclass if the Accept header is set with an explicit
# indent, if an indent level is manually passed, or if you're attempting
# to render `None`.
if accepted_media_type:
base_media_type, params = parse_header(accepted_media_type.encode("ascii"))
indent = params.get("indent", indent)
if data is None or indent:
return super(SuccinctJSONRenderer, self).render(data, accepted_media_type, renderer_context)
return json.dumps(
data, cls=self.encoder_class, indent=indent, ensure_ascii=self.ensure_ascii, separators=(",", ":")
)
def get_filename(self, stream, media_type, parser_context):
"""
Detects the uploaded file name. First searches a "filename" url kwarg.
Then tries to parse Content-Disposition header.
"""
try:
return parser_context["kwargs"]["filename"]
except KeyError:
pass
try:
meta = parser_context["request"].META
disposition = parse_header(meta["HTTP_CONTENT_DISPOSITION"])
return disposition[1]["filename"]
except (AttributeError, KeyError):
pass
def test_rfc2231_wrong_title(self):
"""
Test wrongly formatted RFC 2231 headers (missing double single quotes).
Parsing should not crash (#24209).
"""
test_data = (
(b"Content-Type: application/x-stuff; title*='This%20is%20%2A%2A%2Afun%2A%2A%2A",
b"'This%20is%20%2A%2A%2Afun%2A%2A%2A"),
(b"Content-Type: application/x-stuff; title*='foo.html",
b"'foo.html"),
(b"Content-Type: application/x-stuff; title*=bar.html",
b"bar.html"),
)
for raw_line, expected_title in test_data:
parsed = parse_header(raw_line)
self.assertEqual(parsed[1]['title'], expected_title)
def alter_old_distutils_request(request: WSGIRequest):
"""Alter the request body for compatibility with older distutils clients
Due to a bug in the Python distutils library, the request post is sent
using \n as a separator instead of the \r\n that the HTTP spec demands.
This breaks the Django form parser and therefore we have to write a
custom parser.
This bug was fixed in the Python 2.7.4 and 3.4:
http://bugs.python.org/issue10510
"""
# We first need to retrieve the body before accessing POST or FILES since
# it can only be read once.
body = request.body
if request.POST or request.FILES:
return
new_body = BytesIO()
# Split the response in the various parts based on the boundary string
content_type, opts = parse_header(
request.META['CONTENT_TYPE'].encode('ascii'))
parts = body.split(b'\n--' + opts['boundary'] + b'\n')
for part in parts:
if b'\n\n' not in part:
continue
headers, content = part.split(b'\n\n', 1)
if not headers:
continue
new_body.write(b'--' + opts['boundary'] + b'\r\n')
new_body.write(headers.replace(b'\n', b'\r\n'))
new_body.write(b'\r\n\r\n')
new_body.write(content)
new_body.write(b'\r\n')
new_body.write(b'--' + opts['boundary'] + b'--\r\n')
request._body = new_body.getvalue()
request.META['CONTENT_LENGTH'] = len(request._body)
# Clear out _files and _post so that the request object re-parses the body
if hasattr(request, '_files'):
delattr(request, '_files')
if hasattr(request, '_post'):
delattr(request, '_post')
def _get_indent(self, accepted_media_type, renderer_context):
# If "indent" is provided in the context, then pretty print the result.
# E.g. If we"re being called by the BrowsableAPIRenderer.
renderer_context = renderer_context or {}
indent = renderer_context.get("indent", None)
if accepted_media_type:
# If the media type looks like "application/json; indent=4",
# then pretty print the result.
base_media_type, params = parse_header(accepted_media_type.encode("ascii"))
indent = params.get("indent", indent)
try:
indent = max(min(int(indent), 8), 0)
except (ValueError, TypeError):
indent = None
return indent
def _get_indent(self, accepted_media_type, renderer_context):
# If "indent" is provided in the context, then pretty print the result.
# E.g. If we"re being called by the BrowsableAPIRenderer.
renderer_context = renderer_context or {}
indent = renderer_context.get("indent", None)
if accepted_media_type:
# If the media type looks like "application/json; indent=4",
# then pretty print the result.
base_media_type, params = parse_header(accepted_media_type.encode("ascii"))
indent = params.get("indent", indent)
try:
indent = max(min(int(indent), 8), 0)
except (ValueError, TypeError):
indent = None
return indent
def alter_old_distutils_request(request: WSGIRequest):
"""Alter the request body for compatibility with older distutils clients
Due to a bug in the Python distutils library, the request post is sent
using \n as a separator instead of the \r\n that the HTTP spec demands.
This breaks the Django form parser and therefore we have to write a
custom parser.
This bug was fixed in the Python 2.7.4 and 3.4:
http://bugs.python.org/issue10510
"""
# We first need to retrieve the body before accessing POST or FILES since
# it can only be read once.
body = request.body
if request.POST or request.FILES:
return
new_body = BytesIO()
# Split the response in the various parts based on the boundary string
content_type, opts = parse_header(request.META['CONTENT_TYPE'].encode('ascii'))
parts = body.split(b'\n--' + opts['boundary'] + b'\n')
for part in parts:
if b'\n\n' not in part:
continue
headers, content = part.split(b'\n\n', 1)
if not headers:
continue
new_body.write(b'--' + opts['boundary'] + b'\r\n')
new_body.write(headers.replace(b'\n', b'\r\n'))
new_body.write(b'\r\n\r\n')
new_body.write(content)
new_body.write(b'\r\n')
new_body.write(b'--' + opts['boundary'] + b'--\r\n')
request._body = new_body.getvalue()
request.META['CONTENT_LENGTH'] = len(request._body)
# Clear out _files and _post so that the request object re-parses the body
if hasattr(request, '_files'):
delattr(request, '_files')
if hasattr(request, '_post'):
delattr(request, '_post')
def render(self, data, accepted_media_type=None, renderer_context=None):
renderer_context = renderer_context or {}
indent = renderer_context.get('indent', None)
# Pass to the superclass if the Accept header is set with an explicit
# indent, if an indent level is manually passed, or if you're
# attempting to render `None`.
if accepted_media_type:
base_media_type, params = parse_header(
accepted_media_type.encode('ascii'))
indent = params.get('indent', indent)
if data is None or indent:
return super(SuccinctJSONRenderer, self).render(
data, accepted_media_type, renderer_context)
return json.dumps(
data, cls=self.encoder_class, indent=indent,
ensure_ascii=self.ensure_ascii, separators=(',', ':'))
def test_rfc2231_parsing(self):
test_data = (
(
b"Content-Type: application/x-stuff; "
b"title*=us-ascii'en-us'This%20is%20%2A%2A%2Afun%2A%2A%2A",
"This is ***fun***",
),
(
b"Content-Type: application/x-stuff; title*=UTF-8''foo-%c3%a4.html",
"foo-ä.html",
),
(
b"Content-Type: application/x-stuff; title*=iso-8859-1''foo-%E4.html",
"foo-ä.html",
),
)
for raw_line, expected_title in test_data:
parsed = parse_header(raw_line)
self.assertEqual(parsed[1]["title"], expected_title)
def render(self, data, accepted_media_type=None, renderer_context=None):
"""
Render `obj` into json.
"""
if data is None:
return ''
indent = renderer_context.get('indent', None)
if accepted_media_type:
# If the media type looks like 'application/json; indent=4',
# then pretty print the result.
base_media_type, params = parse_header(accepted_media_type)
indent = params.get('indent', indent)
try:
indent = max(min(int(indent), 8), 0)
except (ValueError, TypeError):
indent = None
return json.dumps(data, cls=self.encoder_class, indent=indent)
def render(self, data, accepted_media_type=None, renderer_context=None):
"""
Render `obj` into json.
"""
if data is None:
return ''
indent = renderer_context.get('indent', None)
if accepted_media_type:
# If the media type looks like 'application/json; indent=4',
# then pretty print the result.
base_media_type, params = parse_header(accepted_media_type)
indent = params.get('indent', indent)
try:
indent = max(min(int(indent), 8), 0)
except (ValueError, TypeError):
indent = None
return json.dumps(data, cls=self.encoder_class, indent=indent)
def get_filename(self, stream, media_type, parser_context):
"""
Detects the uploaded file name. First searches a 'filename' url kwarg.
Then tries to parse Content-Disposition header.
"""
try:
return parser_context["kwargs"]["filename"]
except KeyError:
pass
try:
meta = parser_context["request"].META
disposition = parse_header(
meta["HTTP_CONTENT_DISPOSITION"].encode())
filename_parm = disposition[1]
if "filename*" in filename_parm:
return self.get_encoded_filename(filename_parm)
return force_str(filename_parm["filename"])
except (AttributeError, KeyError, ValueError):
pass
def render(self, data, accepted_media_type=None, renderer_context=None):
'''
Render `data` into JSON, using the faster ujson library for
serialization unless indentation is requested (ujson does not
support indentation).
'''
if data is None:
return bytes()
# If 'indent' is provided in the context, then pretty print the result.
# E.g. If we're being called by the BrowsableAPIRenderer.
renderer_context = renderer_context or {}
indent = renderer_context.get('indent', None)
if accepted_media_type:
# If the media type looks like 'application/json; indent=4',
# then pretty print the result.
base_media_type, params = parse_header(
accepted_media_type.encode('ascii'))
indent = params.get('indent', indent)
try:
indent = max(min(int(indent), 8), 0)
except (ValueError, TypeError):
indent = None
ret = json.dumps(data,
cls=self.encoder_class,
indent=indent,
ensure_ascii=self.ensure_ascii)
else:
ret = ujson.dumps(data, ensure_ascii=self.ensure_ascii)
# On python 2.x json.dumps() returns bytestrings if ensure_ascii=True,
# but if ensure_ascii=False, the return type is underspecified,
# and may (or may not) be unicode.
# On python 3.x json.dumps() returns unicode strings.
if isinstance(ret, six.text_type):
return bytes(ret.encode('utf-8'))
return ret
def render(self, data, accepted_media_type=None, renderer_context=None):
"""
Render `obj` into json.
"""
if data is None:
return ''
# If 'indent' is provided in the context, then pretty print the result.
# E.g. If we're being called by the BrowsableAPIRenderer.
renderer_context = renderer_context or {}
indent = renderer_context.get('indent', None)
if accepted_media_type:
# If the media type looks like 'application/json; indent=4',
# then pretty print the result.
base_media_type, params = parse_header(accepted_media_type.encode('ascii'))
indent = params.get('indent', indent)
try:
indent = max(min(int(indent), 8), 0)
except (ValueError, TypeError):
indent = None
return json.dumps(data, cls=self.encoder_class, indent=indent)
def render(self, data, accepted_media_type=None, renderer_context=None):
'''
Render `data` into JSON, using the faster ujson library for
serialization unless indentation is requested (ujson does not
support indentation).
'''
if data is None:
return bytes()
# If 'indent' is provided in the context, then pretty print the result.
# E.g. If we're being called by the BrowsableAPIRenderer.
renderer_context = renderer_context or {}
indent = renderer_context.get('indent', None)
if accepted_media_type:
# If the media type looks like 'application/json; indent=4',
# then pretty print the result.
base_media_type, params = parse_header(accepted_media_type.encode('ascii'))
indent = params.get('indent', indent)
try:
indent = max(min(int(indent), 8), 0)
except (ValueError, TypeError):
indent = None
ret = json.dumps(data, cls=self.encoder_class,
indent=indent, ensure_ascii=self.ensure_ascii)
else:
ret = ujson.dumps(data, ensure_ascii=self.ensure_ascii)
# On python 2.x json.dumps() returns bytestrings if ensure_ascii=True,
# but if ensure_ascii=False, the return type is underspecified,
# and may (or may not) be unicode.
# On python 3.x json.dumps() returns unicode strings.
if isinstance(ret, six.text_type):
return bytes(ret.encode('utf-8'))
return ret
def __init__(self, media_type_str):
if media_type_str is None:
media_type_str = ''
self.orig = media_type_str
self.full_type, self.params = parse_header(media_type_str)
self.main_type, sep, self.sub_type = self.full_type.partition('/')
def is_json_request(content_type):
"""Return the content type is json.
"""
lhs, rhs = parse_header(content_type)
return lhs == 'application/json'
def __init__(self, media_type_str):
if media_type_str is None:
media_type_str = ''
self.orig = media_type_str
self.full_type, self.params = parse_header(media_type_str.encode(HTTP_HEADER_ENCODING))
self.main_type, sep, self.sub_type = self.full_type.partition('/')
def parse_header(self, media_type):
base_media_type, params = parse_header(media_type)
return params.get(self.version_param, self.default_version)
def __init__(self, media_type_str):
if media_type_str is None:
media_type_str = ''
self.orig = media_type_str
self.full_type, self.params = parse_header(media_type_str)
self.main_type, sep, self.sub_type = self.full_type.partition('/')
def test_parse_header_with_double_quotes_and_semicolon(self):
self.assertEqual(
parse_header(b'form-data; name="files"; filename="fo\\"o;bar"'),
("form-data", {"name": b"files", "filename": b'fo"o;bar'}),
)
def __init__(self, media_type_str):
self.orig = "" if (media_type_str is None) else media_type_str
self.full_type, self.params = parse_header(
self.orig.encode(HTTP_HEADER_ENCODING))
self.main_type, sep, self.sub_type = self.full_type.partition("/")
def fetch_url(
url: str,
response_delegate: typing.Optional[
typing.Callable[[str, Response], typing.Any]
] = None,
user_agent: typing.Optional[str] = None,
seen_urls: typing.Optional[typing.List[str]] = None,
) -> typing.Any:
"""
Fetch a URL, following redirects and denying access to dangerous URLs.
If response_delegate is provided, call out to that function to generate the response and return it. Otherwise, just
return the file name.
"""
# delegate fetching of 3rd party service items that have custom URLs, e.g. Google Docs
service_item = parse_service_url(url)
if service_item:
return fetch_service_item(service_item)
headers = {"User-Agent": user_agent} if user_agent else None
url = convert_raw_content_url(url)
url_obj = urlparse(url)
if url_obj.hostname is None:
raise TypeError('Url "{}" appears to have no hostname.'.format(url))
if is_malicious_host(url_obj.hostname):
raise RemoteFileException(
"{} is not a valid hostname.".format(url_obj.hostname)
)
_, file_ext = splitext(url_obj.path)
if seen_urls is None:
seen_urls = []
with requests.request(
"GET",
url,
headers=headers,
stream=True,
timeout=DOWNLOAD_TIMEOUT_SECONDS,
allow_redirects=False,
) as resp:
if "Location" in resp.headers:
# Manually follow redirects to determine if it is a redirect to a malicious URL
if url in seen_urls:
raise RemoteFileException("Looping redirect found at {}".format(url))
seen_urls.append(url)
if len(seen_urls) > MAX_HTTP_REDIRECTS:
raise RemoteFileException(
"Too many HTTP redirects from original URL: {}".format(seen_urls[0])
)
new_url = urljoin(url, resp.headers["location"])
return fetch_url(new_url, response_delegate, user_agent, seen_urls)
resp.raise_for_status()
file_name = basename(url_obj.path)
if "Content-Disposition" in resp.headers:
content_disposition: typing.Union[str, bytes] = resp.headers[
"Content-Disposition"
]
if not isinstance(content_disposition, bytes):
content_disposition = content_disposition.encode("ascii")
_, metadata = parse_header(content_disposition)
file_name = metadata.get("filename")
if isinstance(file_name, bytes):
file_name = file_name.decode("ascii")
file_name = unquote(file_name)
if response_delegate:
return response_delegate(file_name, resp)
return file_name
def __init__(self, media_type_str):
if media_type_str is None:
media_type_str = ''
self.orig = media_type_str
self.full_type, self.params = parse_header(media_type_str.encode(HTTP_HEADER_ENCODING))
self.main_type, sep, self.sub_type = self.full_type.partition('/')
