def parse(self):
"""
Parse the POST data and break it into a FILES MultiValueDict and a POST
MultiValueDict.
Returns a tuple containing the POST and FILES dictionary, respectively.
"""
encoding = self._encoding
handlers = self._upload_handlers
# HTTP spec says that Content-Length >= 0 is valid
# handling content-length == 0 before continuing
if self._content_length == 0:
return MultiValueDict(), MultiValueDict()
# Create the data structures to be used later.
self._post = MultiValueDict()
# Instantiate the parser and stream:
stream = LazyStream(ChunkIter(self._input_data, self._chunk_size))
# Whether or not to signal a file-completion at the beginning of the loop.
old_field_name = None
counters = [0] * len(handlers)
try:
for item_type, meta_data, field_stream in Parser(stream, self._boundary):
if old_field_name:
# We run this at the beginning of the next loop
# since we cannot be sure a file is complete until
# we hit the next boundary/part of the multipart content.
self.handle_file_complete(old_field_name, counters)
old_field_name = None
try:
disposition = meta_data['content-disposition'][1]
field_name = disposition['name'].strip()
except (KeyError, IndexError, AttributeError):
continue
field_name = force_text(field_name, encoding, errors='replace')
if item_type == FIELD:
# This is a post field, we can just set it in the post
data = field_stream.read()
self._post.appendlist(field_name, data)
elif item_type == FILE:
# This is a file, ignoring...
continue
else:
# If this is neither a FIELD or a FILE, just exhaust the stream.
exhaust(stream)
except StopUpload as e:
if not e.connection_reset:
exhaust(self._input_data)
else:
# Make sure that the request data is all fed
exhaust(self._input_data)
return self._post
def handle_raw_input(
self, input_data, META, content_length, boundary, encoding=None):
"""
Parse the raw input from the HTTP request and split items into fields
and files, executing callback methods as necessary.
Shamelessly adapted and borrowed from
django.http.multiparser.MultiPartParser.
"""
# following suit from the source class, this is imported here to avoid
# a potential circular import
from django.http import QueryDict
# create return values
self.POST = QueryDict('', mutable=True)
self.FILES = MultiValueDict()
# initialize the parser and stream
stream = LazyStream(ChunkIter(input_data, self.chunk_size))
# whether or not to signal a file-completion at the beginning
# of the loop.
old_field_name = None
counter = 0
try:
for item_type, meta_data, field_stream in Parser(stream, boundary):
if old_field_name:
# we run this test at the beginning of the next loop since
# we cannot be sure a file is complete until we hit the
# next boundary/part of the multipart content.
file_obj = self.file_complete(counter)
if file_obj:
# if we return a file object, add it to the files dict
self.FILES.appendlist(force_text(
old_field_name, encoding,
errors='replace'), file_obj)
# wipe it out to prevent havoc
old_field_name = None
try:
disposition = meta_data['content-disposition'][1]
field_name = disposition['name'].strip()
except (KeyError, IndexError, AttributeError):
continue
transfer_encoding = meta_data.get('content-transfer-encoding')
if transfer_encoding is not None:
transfer_encoding = transfer_encoding[0].strip()
field_name = force_text(field_name, encoding, errors='replace')
if item_type == FIELD:
# this is a POST field
if transfer_encoding == "base64":
raw_data = field_stream.read()
try:
data = str(raw_data).decode('base64')
except:
data = raw_data
else:
data = field_stream.read()
self.POST.appendlist(field_name, force_text(
data, encoding, errors='replace'))
# trigger listener
self.field_parsed(field_name, self.POST.get(field_name))
elif item_type == FILE:
# this is a file
file_name = disposition.get('filename')
if not file_name:
continue
# transform the file name
file_name = force_text(
file_name, encoding, errors='replace')
file_name = self.IE_sanitize(unescape_entities(file_name))
content_type = meta_data.get(
'content-type', ('',))[0].strip()
try:
charset = meta_data.get('content-type', (0, {}))[1]\
.get('charset', None)
except:
charset = None
try:
file_content_length = int(
meta_data.get('content-length')[0])
except (IndexError, TypeError, ValueError):
file_content_length = None
counter = 0
# now, do the important file stuff
try:
# alert on the new file
kwargs = {
'content_type': content_type,
'content_length': file_content_length,
'charset': charset}
self.new_file(field_name, file_name, **kwargs)
# chubber-chunk it
for chunk in field_stream:
# we need AES compatibles blocks (multiples of 16 bits)
over_bytes = len(chunk) % 16
if over_bytes:
over_chunk =\
field_stream.read(16 - over_bytes)
chunk += over_chunk
if transfer_encoding == "base64":
try:
chunk = base64.b64decode(chunk)
except Exception as e:
# since this is anly a chunk, any
# error is an unfixable error
raise MultiPartParserError(
"Could not decode base64 data: %r" % e)
chunk_length = len(chunk)
self.receive_data_chunk(chunk, counter)
counter += chunk_length
if counter > settings.UPLOAD_FILE_SIZE_LIMIT:
raise SkipFile('File is too big.')
# ... and we're done
except SkipFile:
# just eat the rest
exhaust(field_stream)
else:
# handle file upload completions on next iteration
old_field_name = field_name
except StopUpload as e:
# if we get a request to stop the upload,
# exhaust it if no con reset
if not e.connection_reset:
exhaust(input_data)
else:
# make sure that the request data is all fed
exhaust(input_data)
# signal the upload has been completed
self.upload_complete()
return self.POST, self.FILES
def handle_upload(request):
if not request.method == "POST":
raise UploadException("AJAX request not valid: must be POST")
if request.is_ajax():
# the file is stored raw in the request
is_raw = True
filename = request.GET.get('qqfile', False) or request.GET.get(
'filename', False) or ''
try:
content_length = int(request.META['CONTENT_LENGTH'])
except (IndexError, TypeError, ValueError):
content_length = None
if content_length < 0:
# This means we shouldn't continue...raise an error.
raise UploadException("Invalid content length: %r" %
content_length)
upload_handlers = request.upload_handlers
for handler in upload_handlers:
handler.handle_raw_input(request, request.META, content_length,
None, None)
pass
# For compatibility with low-level network APIs (with 32-bit integers),
# the chunk size should be < 2^31, but still divisible by 4.
possible_sizes = [
x.chunk_size for x in upload_handlers if x.chunk_size
]
chunk_size = min([2**31 - 4] + possible_sizes)
stream = ChunkIter(request, chunk_size)
counters = [0] * len(upload_handlers)
try:
for handler in upload_handlers:
try:
handler.new_file(None, filename, None, content_length,
None)
except StopFutureHandlers:
break
for chunk in stream:
for i, handler in enumerate(upload_handlers):
chunk_length = len(chunk)
chunk = handler.receive_data_chunk(chunk, counters[i])
counters[i] += chunk_length
if chunk is None:
# If the chunk received by the handler is None, then don't continue.
break
except SkipFile:
# Just use up the rest of this file...
exhaust(stream)
except StopUpload as e:
if not e.connection_reset:
exhaust(request)
else:
# Make sure that the request data is all fed
exhaust(request)
# Signal that the upload has completed.
for handler in upload_handlers:
retval = handler.upload_complete()
if retval:
break
for i, handler in enumerate(upload_handlers):
file_obj = handler.file_complete(counters[i])
if file_obj:
upload = file_obj
break
else:
if len(request.FILES) == 1:
upload, filename, is_raw = handle_request_files_upload(request)
else:
raise UploadException("AJAX request not valid: Bad Upload")
return upload, filename, is_raw
def handle_upload(request):
if not request.method == "POST":
raise UploadException("AJAX request not valid: must be POST")
if request.is_ajax():
# the file is stored raw in the request
is_raw = True
filename = request.GET.get('qqfile', False) or request.GET.get('filename', False) or ''
try:
content_length = int(request.META['CONTENT_LENGTH'])
except (IndexError, TypeError, ValueError):
content_length = None
if content_length < 0:
# This means we shouldn't continue...raise an error.
raise UploadException("Invalid content length: %r" % content_length)
upload_handlers = request.upload_handlers
for handler in upload_handlers:
handler.handle_raw_input(request,
request.META,
content_length,
None,
None)
pass
# For compatibility with low-level network APIs (with 32-bit integers),
# the chunk size should be < 2^31, but still divisible by 4.
possible_sizes = [x.chunk_size for x in upload_handlers if x.chunk_size]
chunk_size = min([2 ** 31 - 4] + possible_sizes)
stream = ChunkIter(request, chunk_size)
counters = [0] * len(upload_handlers)
try:
for handler in upload_handlers:
try:
handler.new_file(None, filename,
None, content_length, None)
except StopFutureHandlers:
break
for chunk in stream:
for i, handler in enumerate(upload_handlers):
chunk_length = len(chunk)
chunk = handler.receive_data_chunk(chunk,
counters[i])
counters[i] += chunk_length
if chunk is None:
# If the chunk received by the handler is None, then don't continue.
break
except SkipFile:
# Just use up the rest of this file...
exhaust(stream)
except StopUpload as e:
if not e.connection_reset:
exhaust(request)
else:
# Make sure that the request data is all fed
exhaust(request)
# Signal that the upload has completed.
for handler in upload_handlers:
retval = handler.upload_complete()
if retval:
break
for i, handler in enumerate(upload_handlers):
file_obj = handler.file_complete(counters[i])
if file_obj:
upload = file_obj
break
else:
if len(request.FILES) == 1:
# FILES is a dictionary in Django but Ajax Upload gives the uploaded file an
# ID based on a random number, so it cannot be guessed here in the code.
# Rather than editing Ajax Upload to pass the ID in the querystring, note that
# each upload is a separate request so FILES should only have one entry.
# Thus, we can just grab the first (and only) value in the dict.
is_raw = False
upload = list(request.FILES.values())[0]
filename = upload.name
else:
raise UploadException("AJAX request not valid: Bad Upload")
return upload, filename, is_raw
def _handle_raw_input_without_file_stream(self,
input_data,
META,
raw_content_length,
boundary,
encoding=None):
"""
Replaces django.http.multipartparser.MultiPartParser.parse
A rfc2388 multipart/form-data parser but replacing the file stream to the creation of empty files.
Returns a tuple of ``(MultiValueDict(POST), MultiValueDict(FILES))``.
"""
# Create the data structures to be used later.
_post = QueryDict(mutable=True)
_files = MultiValueDict()
# For compatibility with low-level network APIs (with 32-bit integers),
# the chunk size should be < 2^31, but still divisible by 4.
_chunk_size = min([2**31 - 4, self.chunk_size])
# Instantiate the parser and stream:
stream = LazyStream(ChunkIter(input_data, _chunk_size))
# Whether or not to signal a file-completion at the beginning of the loop.
old_field_name = None
# Number of bytes that have been read.
num_bytes_read = 0
# To count the number of keys in the request.
num_post_keys = 0
# To limit the amount of data read from the request.
read_size = None
for item_type, meta_data, field_stream in Parser(stream, boundary):
if old_field_name:
# We run this at the beginning of the next loop
# since we cannot be sure a file is complete until
# we hit the next boundary/part of the multipart content.
file_obj = self.file_complete(raw_content_length)
if file_obj:
# If it returns a file object, then set the files dict.
_files.appendlist(
force_str(old_field_name, encoding, errors="replace"),
file_obj)
old_field_name = None
try:
disposition = meta_data["content-disposition"][1]
field_name = disposition["name"].strip()
except (KeyError, IndexError, AttributeError):
continue
transfer_encoding = meta_data.get("content-transfer-encoding")
if transfer_encoding is not None:
transfer_encoding = transfer_encoding[0].strip()
field_name = force_str(field_name, encoding, errors="replace")
if item_type == FIELD:
# NOTE: Parse fields as usual, same as ``MultiPartParser.parse``
# Avoid storing more than DATA_UPLOAD_MAX_NUMBER_FIELDS.
num_post_keys += 1
if (settings.DATA_UPLOAD_MAX_NUMBER_FIELDS is not None
and settings.DATA_UPLOAD_MAX_NUMBER_FIELDS <
num_post_keys):
raise TooManyFieldsSent(
"The number of GET/POST parameters exceeded "
"settings.DATA_UPLOAD_MAX_NUMBER_FIELDS.")
# Avoid reading more than DATA_UPLOAD_MAX_MEMORY_SIZE.
if settings.DATA_UPLOAD_MAX_MEMORY_SIZE is not None:
read_size = settings.DATA_UPLOAD_MAX_MEMORY_SIZE - num_bytes_read
# This is a post field, we can just set it in the post
if transfer_encoding == "base64":
raw_data = field_stream.read(size=read_size)
num_bytes_read += len(raw_data)
try:
data = base64.b64decode(raw_data)
except binascii.Error:
data = raw_data
else:
data = field_stream.read(size=read_size)
num_bytes_read += len(data)
# Add two here to make the check consistent with the
# x-www-form-urlencoded check that includes '&='.
num_bytes_read += len(field_name) + 2
if (settings.DATA_UPLOAD_MAX_MEMORY_SIZE is not None and
num_bytes_read > settings.DATA_UPLOAD_MAX_MEMORY_SIZE):
raise RequestDataTooBig(
"Request body exceeded settings.DATA_UPLOAD_MAX_MEMORY_SIZE."
)
_post.appendlist(field_name,
force_str(data, encoding, errors="replace"))
elif item_type == FILE:
# NOTE: Parse files WITHOUT a stream.
# This is a file, use the handler...
file_name = disposition.get("filename")
if file_name:
file_name = force_str(file_name,
encoding,
errors="replace")
file_name = self.sanitize_file_name(file_name)
if not file_name:
continue
content_type, content_type_extra = meta_data.get(
"content-type", ("", {}))
content_type = content_type.strip()
charset = content_type_extra.get("charset")
content_length = None
self.new_file(field_name, file_name, content_type,
content_length, charset, content_type_extra)
# Handle file upload completions on next iteration.
old_field_name = field_name
else:
# If this is neither a FIELD or a FILE, just exhaust the stream.
exhaust(stream)
# Make sure that the request data is all fed
exhaust(input_data)
_post._mutable = False
return _post, _files
def handle_raw_input(self,
input_data,
META,
content_length,
boundary,
encoding=None):
"""
Parse the raw input from the HTTP request and split items into fields
and files, executing callback methods as necessary.
Shamelessly adapted and borrowed from
django.http.multiparser.MultiPartParser.
"""
# following suit from the source class, this is imported here to avoid
# a potential circular import
from django.http import QueryDict
# create return values
self.POST = QueryDict('', mutable=True)
self.FILES = MultiValueDict()
# initialize the parser and stream
stream = LazyStream(ChunkIter(input_data, self.chunk_size))
# whether or not to signal a file-completion at the beginning
# of the loop.
old_field_name = None
counter = 0
try:
for item_type, meta_data, field_stream in Parser(stream, boundary):
if old_field_name:
# we run this test at the beginning of the next loop since
# we cannot be sure a file is complete until we hit the
# next boundary/part of the multipart content.
file_obj = self.file_complete(counter)
if file_obj:
# if we return a file object, add it to the files dict
self.FILES.appendlist(
force_text(old_field_name,
encoding,
errors='replace'), file_obj)
# wipe it out to prevent havoc
old_field_name = None
try:
disposition = meta_data['content-disposition'][1]
field_name = disposition['name'].strip()
except (KeyError, IndexError, AttributeError):
continue
transfer_encoding = meta_data.get('content-transfer-encoding')
if transfer_encoding is not None:
transfer_encoding = transfer_encoding[0].strip()
field_name = force_text(field_name, encoding, errors='replace')
if item_type == FIELD:
# this is a POST field
if transfer_encoding == "base64":
raw_data = field_stream.read()
try:
data = str(raw_data).decode('base64')
except:
data = raw_data
else:
data = field_stream.read()
self.POST.appendlist(
field_name, force_text(data,
encoding,
errors='replace'))
# trigger listener
self.field_parsed(field_name, self.POST.get(field_name))
elif item_type == FILE:
# this is a file
file_name = disposition.get('filename')
if not file_name:
continue
# transform the file name
file_name = force_text(file_name,
encoding,
errors='replace')
file_name = self.IE_sanitize(unescape_entities(file_name))
content_type = meta_data.get('content-type',
('', ))[0].strip()
try:
charset = meta_data.get('content-type', (0, {}))[1]\
.get('charset', None)
except:
charset = None
try:
file_content_length = int(
meta_data.get('content-length')[0])
except (IndexError, TypeError, ValueError):
file_content_length = None
counter = 0
# now, do the important file stuff
try:
# alert on the new file
kwargs = {
'content_type': content_type,
'content_length': file_content_length,
'charset': charset
}
self.new_file(field_name, file_name, **kwargs)
# chubber-chunk it
for chunk in field_stream:
# we need AES compatibles blocks (multiples of 16 bits)
over_bytes = len(chunk) % 16
if over_bytes:
over_chunk =\
field_stream.read(16 - over_bytes)
chunk += over_chunk
if transfer_encoding == "base64":
try:
chunk = base64.b64decode(chunk)
except Exception as e:
# since this is anly a chunk, any
# error is an unfixable error
raise MultiPartParserError(
"Could not decode base64 data: %r" % e)
chunk_length = len(chunk)
self.receive_data_chunk(chunk, counter)
counter += chunk_length
if counter > settings.UPLOAD_FILE_SIZE_LIMIT:
raise SkipFile('File is too big.')
# ... and we're done
except SkipFile:
# just eat the rest
exhaust(field_stream)
else:
# handle file upload completions on next iteration
old_field_name = field_name
except StopUpload as e:
# if we get a request to stop the upload,
# exhaust it if no con reset
if not e.connection_reset:
exhaust(input_data)
else:
# make sure that the request data is all fed
exhaust(input_data)
# signal the upload has been completed
self.upload_complete()
return self.POST, self.FILES
