def parse(self): """ Parse the POST data and break it into a FILES MultiValueDict and a POST MultiValueDict. Returns a tuple containing the POST and FILES dictionary, respectively. """ encoding = self._encoding handlers = self._upload_handlers # HTTP spec says that Content-Length >= 0 is valid # handling content-length == 0 before continuing if self._content_length == 0: return MultiValueDict(), MultiValueDict() # Create the data structures to be used later. self._post = MultiValueDict() # Instantiate the parser and stream: stream = LazyStream(ChunkIter(self._input_data, self._chunk_size)) # Whether or not to signal a file-completion at the beginning of the loop. old_field_name = None counters = [0] * len(handlers) try: for item_type, meta_data, field_stream in Parser(stream, self._boundary): if old_field_name: # We run this at the beginning of the next loop # since we cannot be sure a file is complete until # we hit the next boundary/part of the multipart content. self.handle_file_complete(old_field_name, counters) old_field_name = None try: disposition = meta_data['content-disposition'][1] field_name = disposition['name'].strip() except (KeyError, IndexError, AttributeError): continue field_name = force_text(field_name, encoding, errors='replace') if item_type == FIELD: # This is a post field, we can just set it in the post data = field_stream.read() self._post.appendlist(field_name, data) elif item_type == FILE: # This is a file, ignoring... continue else: # If this is neither a FIELD or a FILE, just exhaust the stream. exhaust(stream) except StopUpload as e: if not e.connection_reset: exhaust(self._input_data) else: # Make sure that the request data is all fed exhaust(self._input_data) return self._post
def handle_raw_input( self, input_data, META, content_length, boundary, encoding=None): """ Parse the raw input from the HTTP request and split items into fields and files, executing callback methods as necessary. Shamelessly adapted and borrowed from django.http.multiparser.MultiPartParser. """ # following suit from the source class, this is imported here to avoid # a potential circular import from django.http import QueryDict # create return values self.POST = QueryDict('', mutable=True) self.FILES = MultiValueDict() # initialize the parser and stream stream = LazyStream(ChunkIter(input_data, self.chunk_size)) # whether or not to signal a file-completion at the beginning # of the loop. old_field_name = None counter = 0 try: for item_type, meta_data, field_stream in Parser(stream, boundary): if old_field_name: # we run this test at the beginning of the next loop since # we cannot be sure a file is complete until we hit the # next boundary/part of the multipart content. file_obj = self.file_complete(counter) if file_obj: # if we return a file object, add it to the files dict self.FILES.appendlist(force_text( old_field_name, encoding, errors='replace'), file_obj) # wipe it out to prevent havoc old_field_name = None try: disposition = meta_data['content-disposition'][1] field_name = disposition['name'].strip() except (KeyError, IndexError, AttributeError): continue transfer_encoding = meta_data.get('content-transfer-encoding') if transfer_encoding is not None: transfer_encoding = transfer_encoding[0].strip() field_name = force_text(field_name, encoding, errors='replace') if item_type == FIELD: # this is a POST field if transfer_encoding == "base64": raw_data = field_stream.read() try: data = str(raw_data).decode('base64') except: data = raw_data else: data = field_stream.read() self.POST.appendlist(field_name, force_text( data, encoding, errors='replace')) # trigger listener self.field_parsed(field_name, self.POST.get(field_name)) elif item_type == FILE: # this is a file file_name = disposition.get('filename') if not file_name: continue # transform the file name file_name = force_text( file_name, encoding, errors='replace') file_name = self.IE_sanitize(unescape_entities(file_name)) content_type = meta_data.get( 'content-type', ('',))[0].strip() try: charset = meta_data.get('content-type', (0, {}))[1]\ .get('charset', None) except: charset = None try: file_content_length = int( meta_data.get('content-length')[0]) except (IndexError, TypeError, ValueError): file_content_length = None counter = 0 # now, do the important file stuff try: # alert on the new file kwargs = { 'content_type': content_type, 'content_length': file_content_length, 'charset': charset} self.new_file(field_name, file_name, **kwargs) # chubber-chunk it for chunk in field_stream: # we need AES compatibles blocks (multiples of 16 bits) over_bytes = len(chunk) % 16 if over_bytes: over_chunk =\ field_stream.read(16 - over_bytes) chunk += over_chunk if transfer_encoding == "base64": try: chunk = base64.b64decode(chunk) except Exception as e: # since this is anly a chunk, any # error is an unfixable error raise MultiPartParserError( "Could not decode base64 data: %r" % e) chunk_length = len(chunk) self.receive_data_chunk(chunk, counter) counter += chunk_length if counter > settings.UPLOAD_FILE_SIZE_LIMIT: raise SkipFile('File is too big.') # ... and we're done except SkipFile: # just eat the rest exhaust(field_stream) else: # handle file upload completions on next iteration old_field_name = field_name except StopUpload as e: # if we get a request to stop the upload, # exhaust it if no con reset if not e.connection_reset: exhaust(input_data) else: # make sure that the request data is all fed exhaust(input_data) # signal the upload has been completed self.upload_complete() return self.POST, self.FILES
def handle_upload(request): if not request.method == "POST": raise UploadException("AJAX request not valid: must be POST") if request.is_ajax(): # the file is stored raw in the request is_raw = True filename = request.GET.get('qqfile', False) or request.GET.get( 'filename', False) or '' try: content_length = int(request.META['CONTENT_LENGTH']) except (IndexError, TypeError, ValueError): content_length = None if content_length < 0: # This means we shouldn't continue...raise an error. raise UploadException("Invalid content length: %r" % content_length) upload_handlers = request.upload_handlers for handler in upload_handlers: handler.handle_raw_input(request, request.META, content_length, None, None) pass # For compatibility with low-level network APIs (with 32-bit integers), # the chunk size should be < 2^31, but still divisible by 4. possible_sizes = [ x.chunk_size for x in upload_handlers if x.chunk_size ] chunk_size = min([2**31 - 4] + possible_sizes) stream = ChunkIter(request, chunk_size) counters = [0] * len(upload_handlers) try: for handler in upload_handlers: try: handler.new_file(None, filename, None, content_length, None) except StopFutureHandlers: break for chunk in stream: for i, handler in enumerate(upload_handlers): chunk_length = len(chunk) chunk = handler.receive_data_chunk(chunk, counters[i]) counters[i] += chunk_length if chunk is None: # If the chunk received by the handler is None, then don't continue. break except SkipFile: # Just use up the rest of this file... exhaust(stream) except StopUpload as e: if not e.connection_reset: exhaust(request) else: # Make sure that the request data is all fed exhaust(request) # Signal that the upload has completed. for handler in upload_handlers: retval = handler.upload_complete() if retval: break for i, handler in enumerate(upload_handlers): file_obj = handler.file_complete(counters[i]) if file_obj: upload = file_obj break else: if len(request.FILES) == 1: upload, filename, is_raw = handle_request_files_upload(request) else: raise UploadException("AJAX request not valid: Bad Upload") return upload, filename, is_raw
def handle_upload(request): if not request.method == "POST": raise UploadException("AJAX request not valid: must be POST") if request.is_ajax(): # the file is stored raw in the request is_raw = True filename = request.GET.get('qqfile', False) or request.GET.get('filename', False) or '' try: content_length = int(request.META['CONTENT_LENGTH']) except (IndexError, TypeError, ValueError): content_length = None if content_length < 0: # This means we shouldn't continue...raise an error. raise UploadException("Invalid content length: %r" % content_length) upload_handlers = request.upload_handlers for handler in upload_handlers: handler.handle_raw_input(request, request.META, content_length, None, None) pass # For compatibility with low-level network APIs (with 32-bit integers), # the chunk size should be < 2^31, but still divisible by 4. possible_sizes = [x.chunk_size for x in upload_handlers if x.chunk_size] chunk_size = min([2 ** 31 - 4] + possible_sizes) stream = ChunkIter(request, chunk_size) counters = [0] * len(upload_handlers) try: for handler in upload_handlers: try: handler.new_file(None, filename, None, content_length, None) except StopFutureHandlers: break for chunk in stream: for i, handler in enumerate(upload_handlers): chunk_length = len(chunk) chunk = handler.receive_data_chunk(chunk, counters[i]) counters[i] += chunk_length if chunk is None: # If the chunk received by the handler is None, then don't continue. break except SkipFile: # Just use up the rest of this file... exhaust(stream) except StopUpload as e: if not e.connection_reset: exhaust(request) else: # Make sure that the request data is all fed exhaust(request) # Signal that the upload has completed. for handler in upload_handlers: retval = handler.upload_complete() if retval: break for i, handler in enumerate(upload_handlers): file_obj = handler.file_complete(counters[i]) if file_obj: upload = file_obj break else: if len(request.FILES) == 1: # FILES is a dictionary in Django but Ajax Upload gives the uploaded file an # ID based on a random number, so it cannot be guessed here in the code. # Rather than editing Ajax Upload to pass the ID in the querystring, note that # each upload is a separate request so FILES should only have one entry. # Thus, we can just grab the first (and only) value in the dict. is_raw = False upload = list(request.FILES.values())[0] filename = upload.name else: raise UploadException("AJAX request not valid: Bad Upload") return upload, filename, is_raw
def _handle_raw_input_without_file_stream(self, input_data, META, raw_content_length, boundary, encoding=None): """ Replaces django.http.multipartparser.MultiPartParser.parse A rfc2388 multipart/form-data parser but replacing the file stream to the creation of empty files. Returns a tuple of ``(MultiValueDict(POST), MultiValueDict(FILES))``. """ # Create the data structures to be used later. _post = QueryDict(mutable=True) _files = MultiValueDict() # For compatibility with low-level network APIs (with 32-bit integers), # the chunk size should be < 2^31, but still divisible by 4. _chunk_size = min([2**31 - 4, self.chunk_size]) # Instantiate the parser and stream: stream = LazyStream(ChunkIter(input_data, _chunk_size)) # Whether or not to signal a file-completion at the beginning of the loop. old_field_name = None # Number of bytes that have been read. num_bytes_read = 0 # To count the number of keys in the request. num_post_keys = 0 # To limit the amount of data read from the request. read_size = None for item_type, meta_data, field_stream in Parser(stream, boundary): if old_field_name: # We run this at the beginning of the next loop # since we cannot be sure a file is complete until # we hit the next boundary/part of the multipart content. file_obj = self.file_complete(raw_content_length) if file_obj: # If it returns a file object, then set the files dict. _files.appendlist( force_str(old_field_name, encoding, errors="replace"), file_obj) old_field_name = None try: disposition = meta_data["content-disposition"][1] field_name = disposition["name"].strip() except (KeyError, IndexError, AttributeError): continue transfer_encoding = meta_data.get("content-transfer-encoding") if transfer_encoding is not None: transfer_encoding = transfer_encoding[0].strip() field_name = force_str(field_name, encoding, errors="replace") if item_type == FIELD: # NOTE: Parse fields as usual, same as ``MultiPartParser.parse`` # Avoid storing more than DATA_UPLOAD_MAX_NUMBER_FIELDS. num_post_keys += 1 if (settings.DATA_UPLOAD_MAX_NUMBER_FIELDS is not None and settings.DATA_UPLOAD_MAX_NUMBER_FIELDS < num_post_keys): raise TooManyFieldsSent( "The number of GET/POST parameters exceeded " "settings.DATA_UPLOAD_MAX_NUMBER_FIELDS.") # Avoid reading more than DATA_UPLOAD_MAX_MEMORY_SIZE. if settings.DATA_UPLOAD_MAX_MEMORY_SIZE is not None: read_size = settings.DATA_UPLOAD_MAX_MEMORY_SIZE - num_bytes_read # This is a post field, we can just set it in the post if transfer_encoding == "base64": raw_data = field_stream.read(size=read_size) num_bytes_read += len(raw_data) try: data = base64.b64decode(raw_data) except binascii.Error: data = raw_data else: data = field_stream.read(size=read_size) num_bytes_read += len(data) # Add two here to make the check consistent with the # x-www-form-urlencoded check that includes '&='. num_bytes_read += len(field_name) + 2 if (settings.DATA_UPLOAD_MAX_MEMORY_SIZE is not None and num_bytes_read > settings.DATA_UPLOAD_MAX_MEMORY_SIZE): raise RequestDataTooBig( "Request body exceeded settings.DATA_UPLOAD_MAX_MEMORY_SIZE." ) _post.appendlist(field_name, force_str(data, encoding, errors="replace")) elif item_type == FILE: # NOTE: Parse files WITHOUT a stream. # This is a file, use the handler... file_name = disposition.get("filename") if file_name: file_name = force_str(file_name, encoding, errors="replace") file_name = self.sanitize_file_name(file_name) if not file_name: continue content_type, content_type_extra = meta_data.get( "content-type", ("", {})) content_type = content_type.strip() charset = content_type_extra.get("charset") content_length = None self.new_file(field_name, file_name, content_type, content_length, charset, content_type_extra) # Handle file upload completions on next iteration. old_field_name = field_name else: # If this is neither a FIELD or a FILE, just exhaust the stream. exhaust(stream) # Make sure that the request data is all fed exhaust(input_data) _post._mutable = False return _post, _files
def handle_raw_input(self, input_data, META, content_length, boundary, encoding=None): """ Parse the raw input from the HTTP request and split items into fields and files, executing callback methods as necessary. Shamelessly adapted and borrowed from django.http.multiparser.MultiPartParser. """ # following suit from the source class, this is imported here to avoid # a potential circular import from django.http import QueryDict # create return values self.POST = QueryDict('', mutable=True) self.FILES = MultiValueDict() # initialize the parser and stream stream = LazyStream(ChunkIter(input_data, self.chunk_size)) # whether or not to signal a file-completion at the beginning # of the loop. old_field_name = None counter = 0 try: for item_type, meta_data, field_stream in Parser(stream, boundary): if old_field_name: # we run this test at the beginning of the next loop since # we cannot be sure a file is complete until we hit the # next boundary/part of the multipart content. file_obj = self.file_complete(counter) if file_obj: # if we return a file object, add it to the files dict self.FILES.appendlist( force_text(old_field_name, encoding, errors='replace'), file_obj) # wipe it out to prevent havoc old_field_name = None try: disposition = meta_data['content-disposition'][1] field_name = disposition['name'].strip() except (KeyError, IndexError, AttributeError): continue transfer_encoding = meta_data.get('content-transfer-encoding') if transfer_encoding is not None: transfer_encoding = transfer_encoding[0].strip() field_name = force_text(field_name, encoding, errors='replace') if item_type == FIELD: # this is a POST field if transfer_encoding == "base64": raw_data = field_stream.read() try: data = str(raw_data).decode('base64') except: data = raw_data else: data = field_stream.read() self.POST.appendlist( field_name, force_text(data, encoding, errors='replace')) # trigger listener self.field_parsed(field_name, self.POST.get(field_name)) elif item_type == FILE: # this is a file file_name = disposition.get('filename') if not file_name: continue # transform the file name file_name = force_text(file_name, encoding, errors='replace') file_name = self.IE_sanitize(unescape_entities(file_name)) content_type = meta_data.get('content-type', ('', ))[0].strip() try: charset = meta_data.get('content-type', (0, {}))[1]\ .get('charset', None) except: charset = None try: file_content_length = int( meta_data.get('content-length')[0]) except (IndexError, TypeError, ValueError): file_content_length = None counter = 0 # now, do the important file stuff try: # alert on the new file kwargs = { 'content_type': content_type, 'content_length': file_content_length, 'charset': charset } self.new_file(field_name, file_name, **kwargs) # chubber-chunk it for chunk in field_stream: # we need AES compatibles blocks (multiples of 16 bits) over_bytes = len(chunk) % 16 if over_bytes: over_chunk =\ field_stream.read(16 - over_bytes) chunk += over_chunk if transfer_encoding == "base64": try: chunk = base64.b64decode(chunk) except Exception as e: # since this is anly a chunk, any # error is an unfixable error raise MultiPartParserError( "Could not decode base64 data: %r" % e) chunk_length = len(chunk) self.receive_data_chunk(chunk, counter) counter += chunk_length if counter > settings.UPLOAD_FILE_SIZE_LIMIT: raise SkipFile('File is too big.') # ... and we're done except SkipFile: # just eat the rest exhaust(field_stream) else: # handle file upload completions on next iteration old_field_name = field_name except StopUpload as e: # if we get a request to stop the upload, # exhaust it if no con reset if not e.connection_reset: exhaust(input_data) else: # make sure that the request data is all fed exhaust(input_data) # signal the upload has been completed self.upload_complete() return self.POST, self.FILES