def parse(self, stream, media_type=None, parser_context=None): """ Treats the incoming bytestream as a raw file upload and returns a `DateAndFiles` object. `.data` will be None (we expect request body to be a file content). `.files` will be a `QueryDict` containing one "file" element. """ parser_context = parser_context or {} request = parser_context["request"] encoding = parser_context.get("encoding", settings.DEFAULT_CHARSET) meta = request.META upload_handlers = request.upload_handlers filename = self.get_filename(stream, media_type, parser_context) # Note that this code is extracted from Django's handling of # file uploads in MultiPartParser. content_type = meta.get("HTTP_CONTENT_TYPE", meta.get("CONTENT_TYPE", "")) try: content_length = int( meta.get("HTTP_CONTENT_LENGTH", meta.get("CONTENT_LENGTH", 0))) except (ValueError, TypeError): content_length = None # See if the handler will want to take care of the parsing. for handler in upload_handlers: result = handler.handle_raw_input(None, meta, content_length, None, encoding) if result is not None: return DataAndFiles(None, {"file": result[1]}) # This is the standard case. possible_sizes = [ x.chunk_size for x in upload_handlers if x.chunk_size ] chunk_size = min([2**31 - 4] + possible_sizes) chunks = ChunkIter(stream, chunk_size) counters = [0] * len(upload_handlers) for handler in upload_handlers: try: handler.new_file(None, filename, content_type, content_length, encoding) except StopFutureHandlers: break for chunk in chunks: for i, handler in enumerate(upload_handlers): chunk_length = len(chunk) chunk = handler.receive_data_chunk(chunk, counters[i]) counters[i] += chunk_length if chunk is None: break for i, handler in enumerate(upload_handlers): file_obj = handler.file_complete(counters[i]) if file_obj: return DataAndFiles(None, {"file": file_obj}) raise ParseError("FileUpload parse error - " "none of upload handlers can handle the stream")
def handle_upload(request): if not request.method == "POST": raise UploadException("AJAX request not valid: must be POST") if request.is_ajax(): # the file is stored raw in the request is_raw = True filename = request.GET.get('qqfile', False) or request.GET.get( 'filename', False) or '' try: content_length = int(request.META['CONTENT_LENGTH']) except (IndexError, TypeError, ValueError): content_length = None if content_length < 0: # This means we shouldn't continue...raise an error. raise UploadException("Invalid content length: %r" % content_length) upload_handlers = request.upload_handlers for handler in upload_handlers: handler.handle_raw_input(request, request.META, content_length, None, None) pass # For compatibility with low-level network APIs (with 32-bit integers), # the chunk size should be < 2^31, but still divisible by 4. possible_sizes = [ x.chunk_size for x in upload_handlers if x.chunk_size ] chunk_size = min([2**31 - 4] + possible_sizes) stream = ChunkIter(request, chunk_size) counters = [0] * len(upload_handlers) try: for handler in upload_handlers: try: handler.new_file(None, filename, None, content_length, None) except StopFutureHandlers: break for chunk in stream: for i, handler in enumerate(upload_handlers): chunk_length = len(chunk) chunk = handler.receive_data_chunk(chunk, counters[i]) counters[i] += chunk_length if chunk is None: # If the chunk received by the handler is None, then don't continue. break except SkipFile: # Just use up the rest of this file... exhaust(stream) except StopUpload as e: if not e.connection_reset: exhaust(request) else: # Make sure that the request data is all fed exhaust(request) # Signal that the upload has completed. for handler in upload_handlers: retval = handler.upload_complete() if retval: break for i, handler in enumerate(upload_handlers): file_obj = handler.file_complete(counters[i]) if file_obj: upload = file_obj break else: if len(request.FILES) == 1: upload, filename, is_raw = handle_request_files_upload(request) else: raise UploadException("AJAX request not valid: Bad Upload") return upload, filename, is_raw
def parse(self, stream, media_type=None, parser_context=None): '''Treats the incoming bytestream as a raw file upload and returns a `DataAndFiles` object. `.data` will be None (we expect request body to be a file content). `.files` will be a `QueryDict` containing one 'file' element. ''' parser_context = parser_context or {} request = parser_context['request'] encoding = parser_context.get('encoding', settings.DEFAULT_CHARSET) meta = request.META upload_handlers = request.upload_handlers filename = "%s.sif" % (str(uuid.uuid4())) print("Filename for parser %s" % filename) # Note that this code is extracted from Django's handling of # file uploads in MultiPartParser. content_type = meta.get('HTTP_CONTENT_TYPE', meta.get('CONTENT_TYPE', '')) try: content_length = int( meta.get('HTTP_CONTENT_LENGTH', meta.get('CONTENT_LENGTH', 0))) except (ValueError, TypeError): content_length = None # See if the handler will want to take care of the parsing. for handler in upload_handlers: result = handler.handle_raw_input(stream, meta, content_length, None, encoding) if result is not None: return DataAndFiles({}, {'file': result[1]}) # This is the standard case. possible_sizes = [ x.chunk_size for x in upload_handlers if x.chunk_size ] chunk_size = min([2**31 - 4] + possible_sizes) chunks = ChunkIter(stream, chunk_size) counters = [0] * len(upload_handlers) for index, handler in enumerate(upload_handlers): try: handler.new_file(None, filename, content_type, content_length, encoding) except StopFutureHandlers: upload_handlers = upload_handlers[:index + 1] break for chunk in chunks: for index, handler in enumerate(upload_handlers): chunk_length = len(chunk) chunk = handler.receive_data_chunk(chunk, counters[index]) counters[index] += chunk_length if chunk is None: break for index, handler in enumerate(upload_handlers): file_obj = handler.file_complete(counters[index]) if file_obj is not None: return DataAndFiles({}, {'file': file_obj}) raise ParseError(self.errors['unhandled'])
def _handle_raw_input_without_file_stream(self, input_data, META, raw_content_length, boundary, encoding=None): """ Replaces django.http.multipartparser.MultiPartParser.parse A rfc2388 multipart/form-data parser but replacing the file stream to the creation of empty files. Returns a tuple of ``(MultiValueDict(POST), MultiValueDict(FILES))``. """ # Create the data structures to be used later. _post = QueryDict(mutable=True) _files = MultiValueDict() # For compatibility with low-level network APIs (with 32-bit integers), # the chunk size should be < 2^31, but still divisible by 4. _chunk_size = min([2**31 - 4, self.chunk_size]) # Instantiate the parser and stream: stream = LazyStream(ChunkIter(input_data, _chunk_size)) # Whether or not to signal a file-completion at the beginning of the loop. old_field_name = None # Number of bytes that have been read. num_bytes_read = 0 # To count the number of keys in the request. num_post_keys = 0 # To limit the amount of data read from the request. read_size = None for item_type, meta_data, field_stream in Parser(stream, boundary): if old_field_name: # We run this at the beginning of the next loop # since we cannot be sure a file is complete until # we hit the next boundary/part of the multipart content. file_obj = self.file_complete(raw_content_length) if file_obj: # If it returns a file object, then set the files dict. _files.appendlist( force_str(old_field_name, encoding, errors="replace"), file_obj) old_field_name = None try: disposition = meta_data["content-disposition"][1] field_name = disposition["name"].strip() except (KeyError, IndexError, AttributeError): continue transfer_encoding = meta_data.get("content-transfer-encoding") if transfer_encoding is not None: transfer_encoding = transfer_encoding[0].strip() field_name = force_str(field_name, encoding, errors="replace") if item_type == FIELD: # NOTE: Parse fields as usual, same as ``MultiPartParser.parse`` # Avoid storing more than DATA_UPLOAD_MAX_NUMBER_FIELDS. num_post_keys += 1 if (settings.DATA_UPLOAD_MAX_NUMBER_FIELDS is not None and settings.DATA_UPLOAD_MAX_NUMBER_FIELDS < num_post_keys): raise TooManyFieldsSent( "The number of GET/POST parameters exceeded " "settings.DATA_UPLOAD_MAX_NUMBER_FIELDS.") # Avoid reading more than DATA_UPLOAD_MAX_MEMORY_SIZE. if settings.DATA_UPLOAD_MAX_MEMORY_SIZE is not None: read_size = settings.DATA_UPLOAD_MAX_MEMORY_SIZE - num_bytes_read # This is a post field, we can just set it in the post if transfer_encoding == "base64": raw_data = field_stream.read(size=read_size) num_bytes_read += len(raw_data) try: data = base64.b64decode(raw_data) except binascii.Error: data = raw_data else: data = field_stream.read(size=read_size) num_bytes_read += len(data) # Add two here to make the check consistent with the # x-www-form-urlencoded check that includes '&='. num_bytes_read += len(field_name) + 2 if (settings.DATA_UPLOAD_MAX_MEMORY_SIZE is not None and num_bytes_read > settings.DATA_UPLOAD_MAX_MEMORY_SIZE): raise RequestDataTooBig( "Request body exceeded settings.DATA_UPLOAD_MAX_MEMORY_SIZE." ) _post.appendlist(field_name, force_str(data, encoding, errors="replace")) elif item_type == FILE: # NOTE: Parse files WITHOUT a stream. # This is a file, use the handler... file_name = disposition.get("filename") if file_name: file_name = force_str(file_name, encoding, errors="replace") file_name = self.sanitize_file_name(file_name) if not file_name: continue content_type, content_type_extra = meta_data.get( "content-type", ("", {})) content_type = content_type.strip() charset = content_type_extra.get("charset") content_length = None self.new_file(field_name, file_name, content_type, content_length, charset, content_type_extra) # Handle file upload completions on next iteration. old_field_name = field_name else: # If this is neither a FIELD or a FILE, just exhaust the stream. exhaust(stream) # Make sure that the request data is all fed exhaust(input_data) _post._mutable = False return _post, _files
def handle_raw_input(self, input_data, META, content_length, boundary, encoding=None): """ Parse the raw input from the HTTP request and split items into fields and files, executing callback methods as necessary. Shamelessly adapted and borrowed from django.http.multiparser.MultiPartParser. """ # following suit from the source class, this is imported here to avoid # a potential circular import from django.http import QueryDict # create return values self.POST = QueryDict('', mutable=True) self.FILES = MultiValueDict() # initialize the parser and stream stream = LazyStream(ChunkIter(input_data, self.chunk_size)) # whether or not to signal a file-completion at the beginning # of the loop. old_field_name = None counter = 0 try: for item_type, meta_data, field_stream in Parser(stream, boundary): if old_field_name: # we run this test at the beginning of the next loop since # we cannot be sure a file is complete until we hit the # next boundary/part of the multipart content. file_obj = self.file_complete(counter) if file_obj: # if we return a file object, add it to the files dict self.FILES.appendlist( force_text(old_field_name, encoding, errors='replace'), file_obj) # wipe it out to prevent havoc old_field_name = None try: disposition = meta_data['content-disposition'][1] field_name = disposition['name'].strip() except (KeyError, IndexError, AttributeError): continue transfer_encoding = meta_data.get('content-transfer-encoding') if transfer_encoding is not None: transfer_encoding = transfer_encoding[0].strip() field_name = force_text(field_name, encoding, errors='replace') if item_type == FIELD: # this is a POST field if transfer_encoding == "base64": raw_data = field_stream.read() try: data = str(raw_data).decode('base64') except: data = raw_data else: data = field_stream.read() self.POST.appendlist( field_name, force_text(data, encoding, errors='replace')) # trigger listener self.field_parsed(field_name, self.POST.get(field_name)) elif item_type == FILE: # this is a file file_name = disposition.get('filename') if not file_name: continue # transform the file name file_name = force_text(file_name, encoding, errors='replace') file_name = self.IE_sanitize(unescape_entities(file_name)) content_type = meta_data.get('content-type', ('', ))[0].strip() try: charset = meta_data.get('content-type', (0, {}))[1]\ .get('charset', None) except: charset = None try: file_content_length = int( meta_data.get('content-length')[0]) except (IndexError, TypeError, ValueError): file_content_length = None counter = 0 # now, do the important file stuff try: # alert on the new file kwargs = { 'content_type': content_type, 'content_length': file_content_length, 'charset': charset } self.new_file(field_name, file_name, **kwargs) # chubber-chunk it for chunk in field_stream: # we need AES compatibles blocks (multiples of 16 bits) over_bytes = len(chunk) % 16 if over_bytes: over_chunk =\ field_stream.read(16 - over_bytes) chunk += over_chunk if transfer_encoding == "base64": try: chunk = base64.b64decode(chunk) except Exception as e: # since this is anly a chunk, any # error is an unfixable error raise MultiPartParserError( "Could not decode base64 data: %r" % e) chunk_length = len(chunk) self.receive_data_chunk(chunk, counter) counter += chunk_length if counter > settings.UPLOAD_FILE_SIZE_LIMIT: raise SkipFile('File is too big.') # ... and we're done except SkipFile: # just eat the rest exhaust(field_stream) else: # handle file upload completions on next iteration old_field_name = field_name except StopUpload as e: # if we get a request to stop the upload, # exhaust it if no con reset if not e.connection_reset: exhaust(input_data) else: # make sure that the request data is all fed exhaust(input_data) # signal the upload has been completed self.upload_complete() return self.POST, self.FILES