async def read_body(self, read):
"""
Reads a HTTP body from an ASGI connection.
"""
body = b""
while True:
message = await receive()
if message["type"] == "http.disconnect":
# Bye bye.
raise RequestAborted()
else:
# See if the message has body, and if it's the end, launch into
# handling (and a synchronous subthread)
if "body" in message:
body += message["body"]
if not message.get("more_body", False):
break
# Limit the maximum request data size that will be handled in-memory.
# TODO: Stream the body to temp disk instead?
# (we can't provide a file-like with direct reading as we would not be async)
if (settings.DATA_UPLOAD_MAX_MEMORY_SIZE is not None and
self._content_length > settings.DATA_UPLOAD_MAX_MEMORY_SIZE):
raise RequestDataTooBig(
"Request body exceeded settings.DATA_UPLOAD_MAX_MEMORY_SIZE.")
return body
def result(request):
# t1 = time.time()
query = request.GET.get('searchword')
# TODO: validator 작성? [a-zA-Aㄱ-힣0-9-_!@#$%&*()=+.,/?'";:[]{}~₩] // &# 비허용
if not query:
raise Http404()
if len(query.encode("utf8")) > 100: # ring에서 prefix를 붙이므로 적절히 100으로 한다
raise RequestDataTooBig(
'Memcached key length must be less than 100 bytes')
key = query.strip().replace(" ", ".")
total_resultset = search_books(key)
page = request.GET.get('page')
paginator = Paginator(total_resultset, 20)
try:
total_resultset_paged = paginator.page(page)
except PageNotAnInteger:
total_resultset_paged = paginator.page(1)
except EmptyPage:
total_resultset_paged = paginator.page(paginator.num_pages)
context = {
'total_resultset_paged': total_resultset_paged,
'yes24_url': yes24_base_url + quote(query, encoding="euc-kr"),
'aladin_url': aladin_base_url + quote(query, encoding="euc-kr"),
}
# print(time.time()-t1, '초, 총 소요시간')
return render(request, 'moa/search_result.html', context)
def create(self, validated_data):
"""
Create report instance and submit it to civicrm server, given the validated data.
"""
instance = Report()
try: #loading the encryption key
report_decryptor = ReportDecryptor(
server_settings.RECEIVER_SECRET_KEY_FILENAME,
receiver_key_id=validated_data.get('encryption_key_id'))
except RuntimeError as e:
raise RuntimeError("unable to retrieve the report encryption key")
#the correct way of doing this to invoke the serializer
#of the report here for validation
try:
decrypted_report = json.loads(
report_decryptor.decrypt_report(
validated_data.get('encrypted_blob')))
except:
raise RuntimeError("submitted Encrypted blob is badly formated")
if ((not 'client_id' in decrypted_report.keys())
or (not 'report_body' in decrypted_report.keys())):
raise RuntimeError("both client id and report body are required")
instance.client_id = decrypted_report['client_id']
#first we check if the request is coming from a valid client
self._validate_client_id(instance.client_id)
#now that we have a valid client_id we retrieve client_session
#make sure the client has not passed their submission quota
client_record = self._get_client_submission_record(instance.client_id)
today_key = datetime.datetime.today().strftime('%Y-%m-%d')
self._verify_client_no_of_submission_quota_usage(
client_record, today_key)
try:
instance.submission_time = validated_data.get('submission_time')
instance.report_id = decrypted_report['report_id']
instance.report_body = decrypted_report['report_body']
instance.reporter_name = decrypted_report['name']
instance.reporter_email = decrypted_report['email']
instance.reporter_telegram = decrypted_report['telegram']
except:
raise RuntimeError("Error in the set of submitted fields")
#reject if the report is too large
#all other fields are size limited in civicrm so the repor_body is the only concern
if (len(decrypted_report['report_body']) >
server_settings.MAX_SIZE_OF_SUBMISSION):
raise RequestDataTooBig("too big of a report")
instance.report_body = decrypted_report['report_body']
instance.save()
#counting client's daily submission
client_record[today_key + "_no"] += 1
client_record.save()
return instance
def check_content_length(parser_context):
if parser_context and settings.DATA_UPLOAD_MAX_MEMORY_SIZE and 'request' in parser_context:
try:
content_length = int(parser_context['request'].META.get(
'CONTENT_LENGTH', 0))
except (ValueError, TypeError):
content_length = 0
if content_length and content_length > settings.DATA_UPLOAD_MAX_MEMORY_SIZE or content_length < 0:
raise RequestDataTooBig(
'Request body exceeded settings.DATA_UPLOAD_MAX_MEMORY_SIZE.')
def handle_raw_input(self,
input_data,
META,
content_length,
boundary,
encoding=None):
"""
Handle the raw input from the client.
"""
max_size = getattr(settings, 'CUSTOM_UPLOAD_MAX_FILE_SIZE',
10 * 2**30) # default 10GB
if max_size is None:
return
if content_length > max_size:
raise RequestDataTooBig(gettext('上传文件超过大小限制'))
def body(self):
if not hasattr(self, '_body'):
if self._read_started:
raise RawPostDataException("You cannot access body after reading from request's data stream")
# Limit the maximum request data size that will be handled in-memory.
if (settings.DATA_UPLOAD_MAX_MEMORY_SIZE is not None and
int(self.META.get('CONTENT_LENGTH') or 0) > settings.DATA_UPLOAD_MAX_MEMORY_SIZE):
raise RequestDataTooBig('Request body exceeded settings.DATA_UPLOAD_MAX_MEMORY_SIZE.')
try:
self._body = self.read()
except IOError as e:
raise UnreadablePostError(*e.args) from e
self._stream = BytesIO(self._body)
return self._body
def handle_raw_input(self,
input_data,
META,
content_length,
boundary,
encoding=None):
"""
Handle the raw input from the client.
"""
max_size = self.get_max_size_upload_limit()
if max_size is None:
return
if content_length > max_size:
raise RequestDataTooBig(gettext('上传文件超过大小限制'))
if content_length <= 0:
raise Exception(gettext('无效的标头Content-Length'))
def parse(self):
"""
Parse the POST data and break it into a FILES MultiValueDict and a POST
MultiValueDict.
Return a tuple containing the POST and FILES dictionary, respectively.
"""
from django.http import QueryDict
encoding = self._encoding
handlers = self._upload_handlers
# HTTP spec says that Content-Length >= 0 is valid
# handling content-length == 0 before continuing
if self._content_length == 0:
return QueryDict(encoding=self._encoding), MultiValueDict()
# See if any of the handlers take care of the parsing.
# This allows overriding everything if need be.
for handler in handlers:
result = handler.handle_raw_input(
self._input_data,
self._meta,
self._content_length,
self._boundary,
encoding,
)
# Check to see if it was handled
if result is not None:
return result[0], result[1]
# Create the data structures to be used later.
self._post = QueryDict(mutable=True)
self._files = MultiValueDict()
# Instantiate the parser and stream:
stream = LazyStream(ChunkIter(self._input_data, self._chunk_size))
# Whether or not to signal a file-completion at the beginning of the loop.
old_field_name = None
counters = [0] * len(handlers)
# Number of bytes that have been read.
num_bytes_read = 0
# To count the number of keys in the request.
num_post_keys = 0
# To limit the amount of data read from the request.
read_size = None
try:
for item_type, meta_data, field_stream in Parser(
stream, self._boundary):
if old_field_name:
# We run this at the beginning of the next loop
# since we cannot be sure a file is complete until
# we hit the next boundary/part of the multipart content.
self.handle_file_complete(old_field_name, counters)
old_field_name = None
try:
disposition = meta_data['content-disposition'][1]
field_name = disposition['name'].strip()
except (KeyError, IndexError, AttributeError):
continue
transfer_encoding = meta_data.get('content-transfer-encoding')
if transfer_encoding is not None:
transfer_encoding = transfer_encoding[0].strip()
field_name = force_str(field_name, encoding, errors='replace')
if item_type == FIELD:
# Avoid storing more than DATA_UPLOAD_MAX_NUMBER_FIELDS.
num_post_keys += 1
if (settings.DATA_UPLOAD_MAX_NUMBER_FIELDS is not None
and settings.DATA_UPLOAD_MAX_NUMBER_FIELDS <
num_post_keys):
raise TooManyFieldsSent(
'The number of GET/POST parameters exceeded '
'settings.DATA_UPLOAD_MAX_NUMBER_FIELDS.')
# Avoid reading more than DATA_UPLOAD_MAX_MEMORY_SIZE.
if settings.DATA_UPLOAD_MAX_MEMORY_SIZE is not None:
read_size = settings.DATA_UPLOAD_MAX_MEMORY_SIZE - num_bytes_read
# This is a post field, we can just set it in the post
if transfer_encoding == 'base64':
raw_data = field_stream.read(size=read_size)
num_bytes_read += len(raw_data)
try:
data = base64.b64decode(raw_data)
except binascii.Error:
data = raw_data
else:
data = field_stream.read(size=read_size)
num_bytes_read += len(data)
# Add two here to make the check consistent with the
# x-www-form-urlencoded check that includes '&='.
num_bytes_read += len(field_name) + 2
if (settings.DATA_UPLOAD_MAX_MEMORY_SIZE is not None
and num_bytes_read >
settings.DATA_UPLOAD_MAX_MEMORY_SIZE):
raise RequestDataTooBig(
'Request body exceeded settings.DATA_UPLOAD_MAX_MEMORY_SIZE.'
)
self._post.appendlist(
field_name, force_str(data, encoding,
errors='replace'))
elif item_type == FILE:
# This is a file, use the handler...
file_name = disposition.get('filename')
if file_name:
file_name = force_str(file_name,
encoding,
errors='replace')
file_name = self.IE_sanitize(html.unescape(file_name))
if not file_name:
continue
content_type, content_type_extra = meta_data.get(
'content-type', ('', {}))
content_type = content_type.strip()
charset = content_type_extra.get('charset')
try:
content_length = int(
meta_data.get('content-length')[0])
except (IndexError, TypeError, ValueError):
content_length = None
counters = [0] * len(handlers)
try:
for handler in handlers:
try:
handler.new_file(
field_name,
file_name,
content_type,
content_length,
charset,
content_type_extra,
)
except StopFutureHandlers:
break
for chunk in field_stream:
if transfer_encoding == 'base64':
# We only special-case base64 transfer encoding
# We should always decode base64 chunks by multiple of 4,
# ignoring whitespace.
stripped_chunk = b"".join(chunk.split())
remaining = len(stripped_chunk) % 4
while remaining != 0:
over_chunk = field_stream.read(4 -
remaining)
stripped_chunk += b"".join(
over_chunk.split())
remaining = len(stripped_chunk) % 4
try:
chunk = base64.b64decode(stripped_chunk)
except Exception as exc:
# Since this is only a chunk, any error is an unfixable error.
raise MultiPartParserError(
"Could not decode base64 data."
) from exc
for i, handler in enumerate(handlers):
chunk_length = len(chunk)
chunk = handler.receive_data_chunk(
chunk, counters[i])
counters[i] += chunk_length
if chunk is None:
# Don't continue if the chunk received by
# the handler is None.
break
except SkipFile:
self._close_files()
# Just use up the rest of this file...
exhaust(field_stream)
else:
# Handle file upload completions on next iteration.
old_field_name = field_name
else:
# If this is neither a FIELD or a FILE, just exhaust the stream.
exhaust(stream)
except StopUpload as e:
self._close_files()
if not e.connection_reset:
exhaust(self._input_data)
else:
# Make sure that the request data is all fed
exhaust(self._input_data)
# Signal that the upload has completed.
# any() shortcircuits if a handler's upload_complete() returns a value.
any(handler.upload_complete() for handler in handlers)
self._post._mutable = False
return self._post, self._files
def _handle_raw_input_without_file_stream(self,
input_data,
META,
raw_content_length,
boundary,
encoding=None):
"""
Replaces django.http.multipartparser.MultiPartParser.parse
A rfc2388 multipart/form-data parser but replacing the file stream to the creation of empty files.
Returns a tuple of ``(MultiValueDict(POST), MultiValueDict(FILES))``.
"""
# Create the data structures to be used later.
_post = QueryDict(mutable=True)
_files = MultiValueDict()
# For compatibility with low-level network APIs (with 32-bit integers),
# the chunk size should be < 2^31, but still divisible by 4.
_chunk_size = min([2**31 - 4, self.chunk_size])
# Instantiate the parser and stream:
stream = LazyStream(ChunkIter(input_data, _chunk_size))
# Whether or not to signal a file-completion at the beginning of the loop.
old_field_name = None
# Number of bytes that have been read.
num_bytes_read = 0
# To count the number of keys in the request.
num_post_keys = 0
# To limit the amount of data read from the request.
read_size = None
for item_type, meta_data, field_stream in Parser(stream, boundary):
if old_field_name:
# We run this at the beginning of the next loop
# since we cannot be sure a file is complete until
# we hit the next boundary/part of the multipart content.
file_obj = self.file_complete(raw_content_length)
if file_obj:
# If it returns a file object, then set the files dict.
_files.appendlist(
force_str(old_field_name, encoding, errors="replace"),
file_obj)
old_field_name = None
try:
disposition = meta_data["content-disposition"][1]
field_name = disposition["name"].strip()
except (KeyError, IndexError, AttributeError):
continue
transfer_encoding = meta_data.get("content-transfer-encoding")
if transfer_encoding is not None:
transfer_encoding = transfer_encoding[0].strip()
field_name = force_str(field_name, encoding, errors="replace")
if item_type == FIELD:
# NOTE: Parse fields as usual, same as ``MultiPartParser.parse``
# Avoid storing more than DATA_UPLOAD_MAX_NUMBER_FIELDS.
num_post_keys += 1
if (settings.DATA_UPLOAD_MAX_NUMBER_FIELDS is not None
and settings.DATA_UPLOAD_MAX_NUMBER_FIELDS <
num_post_keys):
raise TooManyFieldsSent(
"The number of GET/POST parameters exceeded "
"settings.DATA_UPLOAD_MAX_NUMBER_FIELDS.")
# Avoid reading more than DATA_UPLOAD_MAX_MEMORY_SIZE.
if settings.DATA_UPLOAD_MAX_MEMORY_SIZE is not None:
read_size = settings.DATA_UPLOAD_MAX_MEMORY_SIZE - num_bytes_read
# This is a post field, we can just set it in the post
if transfer_encoding == "base64":
raw_data = field_stream.read(size=read_size)
num_bytes_read += len(raw_data)
try:
data = base64.b64decode(raw_data)
except binascii.Error:
data = raw_data
else:
data = field_stream.read(size=read_size)
num_bytes_read += len(data)
# Add two here to make the check consistent with the
# x-www-form-urlencoded check that includes '&='.
num_bytes_read += len(field_name) + 2
if (settings.DATA_UPLOAD_MAX_MEMORY_SIZE is not None and
num_bytes_read > settings.DATA_UPLOAD_MAX_MEMORY_SIZE):
raise RequestDataTooBig(
"Request body exceeded settings.DATA_UPLOAD_MAX_MEMORY_SIZE."
)
_post.appendlist(field_name,
force_str(data, encoding, errors="replace"))
elif item_type == FILE:
# NOTE: Parse files WITHOUT a stream.
# This is a file, use the handler...
file_name = disposition.get("filename")
if file_name:
file_name = force_str(file_name,
encoding,
errors="replace")
file_name = self.sanitize_file_name(file_name)
if not file_name:
continue
content_type, content_type_extra = meta_data.get(
"content-type", ("", {}))
content_type = content_type.strip()
charset = content_type_extra.get("charset")
content_length = None
self.new_file(field_name, file_name, content_type,
content_length, charset, content_type_extra)
# Handle file upload completions on next iteration.
old_field_name = field_name
else:
# If this is neither a FIELD or a FILE, just exhaust the stream.
exhaust(stream)
# Make sure that the request data is all fed
exhaust(input_data)
_post._mutable = False
return _post, _files
def __init__(self, scope, body):
self.scope = scope
self._content_length = 0
self._post_parse_error = False
self._read_started = False
self.resolver_match = None
self.script_name = self.scope.get("root_path", "")
if self.script_name and scope["path"].startswith(self.script_name):
# TODO: Better is-prefix checking, slash handling?
self.path_info = scope["path"][len(self.script_name) :]
else:
self.path_info = scope["path"]
# django path is different from asgi scope path args, it should combine with script name
if self.script_name:
self.path = "%s/%s" % (
self.script_name.rstrip("/"),
self.path_info.replace("/", "", 1),
)
else:
self.path = scope["path"]
# HTTP basics
self.method = self.scope["method"].upper()
# fix https://github.com/django/channels/issues/622
query_string = self.scope.get("query_string", "")
if isinstance(query_string, bytes):
query_string = query_string.decode("utf-8")
self.META = {
"REQUEST_METHOD": self.method,
"QUERY_STRING": query_string,
"SCRIPT_NAME": self.script_name,
"PATH_INFO": self.path_info,
# Old code will need these for a while
"wsgi.multithread": True,
"wsgi.multiprocess": True,
}
if self.scope.get("client", None):
self.META["REMOTE_ADDR"] = self.scope["client"][0]
self.META["REMOTE_HOST"] = self.META["REMOTE_ADDR"]
self.META["REMOTE_PORT"] = self.scope["client"][1]
if self.scope.get("server", None):
self.META["SERVER_NAME"] = self.scope["server"][0]
self.META["SERVER_PORT"] = str(self.scope["server"][1])
else:
self.META["SERVER_NAME"] = "unknown"
self.META["SERVER_PORT"] = "0"
# Handle old style-headers for a transition period
if "headers" in self.scope and isinstance(self.scope["headers"], dict):
self.scope["headers"] = [
(x.encode("latin1"), y) for x, y in self.scope["headers"].items()
]
# Headers go into META
for name, value in self.scope.get("headers", []):
name = name.decode("latin1")
if name == "content-length":
corrected_name = "CONTENT_LENGTH"
elif name == "content-type":
corrected_name = "CONTENT_TYPE"
else:
corrected_name = "HTTP_%s" % name.upper().replace("-", "_")
# HTTPbis say only ASCII chars are allowed in headers, but we latin1 just in case
value = value.decode("latin1")
if corrected_name in self.META:
value = self.META[corrected_name] + "," + value
self.META[corrected_name] = value
# Pull out request encoding if we find it
if "CONTENT_TYPE" in self.META:
self.content_type, self.content_params = cgi.parse_header(
self.META["CONTENT_TYPE"]
)
if "charset" in self.content_params:
try:
codecs.lookup(self.content_params["charset"])
except LookupError:
pass
else:
self.encoding = self.content_params["charset"]
else:
self.content_type, self.content_params = "", {}
# Pull out content length info
if self.META.get("CONTENT_LENGTH", None):
try:
self._content_length = int(self.META["CONTENT_LENGTH"])
except (ValueError, TypeError):
pass
# Body handling
# TODO: chunked bodies
# Limit the maximum request data size that will be handled in-memory.
if (
settings.DATA_UPLOAD_MAX_MEMORY_SIZE is not None
and self._content_length > settings.DATA_UPLOAD_MAX_MEMORY_SIZE
):
raise RequestDataTooBig(
"Request body exceeded settings.DATA_UPLOAD_MAX_MEMORY_SIZE."
)
self._body = body
assert isinstance(self._body, bytes), "Body is not bytes"
# Add a stream-a-like for the body
self._stream = BytesIO(self._body)
# Other bits
self.resolver_match = None
def create(self, validated_data):
"""
Create and return a new `Snippet` instance, given the validated data.
"""
instance = ReportAttachment()
try:
attachment_decryptor = ReportDecryptor(
server_settings.RECEIVER_SECRET_KEY_FILENAME,
receiver_key_id=validated_data.get('encryption_kdey_id'))
except RuntimeError as e:
raise RuntimeError("unable to retrieve the report encryption key")
try:
decrypted_attachment = json.loads(
attachment_decryptor.decrypt_report(
validated_data.get('encrypted_blob')))
except:
raise RuntimeError("submitted Encrypted blob is badly formated")
if (not 'client_id' in decrypted_attachment.keys()):
raise RuntimeError("client id is required")
#first we check if the request is coming from a valid client
instance.client_id = decrypted_attachment['client_id']
self._validate_client_id(instance.client_id)
try:
instance.attachment_id = decrypted_attachment['attachment_id']
instance.report_id = decrypted_attachment['report_id']
instance.submission_time = validated_data.get('submission_time')
instance.attachment_type = decrypted_attachment['attachment_type']
except:
raise RuntimeError("Error in the set of submitted fields")
if (validated_data.get('attachment_data')):
instance.attachment_data = self._decrypt_attachement(
decrypted_attachment['encryption_key'],
decrypted_attachment['encryption_iv'],
validated_data.get('attachment_data').read())
elif (validated_data.get('s3_submission') == True):
data_retreival_response = self._retrieve_attachment_from_s3(
instance.attachment_id)
if data_retreival_response.status_code != 200:
raise RuntimeError(
'Failed to retrieve attachment data from S3 due to Error %s'
% data_retreival_response.status_code)
instance.attachment_data = self._decrypt_attachement(
decrypted_attachment['encryption_key'],
decrypted_attachment['encryption_iv'],
data_retreival_response.content)
else:
raise RuntimeError(
"submission includes attachment meta data without body")
current_submission_size = len(instance.attachment_data)
if (current_submission_size > server_settings.MAX_SIZE_OF_SUBMISSION):
raise RequestDataTooBig("too big of a attachment")
#now that we have a valid client_id we retrieve client_session
#make sure the client has not passed their size of submission quota
#otherwise raise an exception
client_record = self._get_client_submission_record(instance.client_id)
today_key = datetime.datetime.today().strftime('%Y-%m-%d')
self._verify_client_size_of_submission_quota_usage(
client_record, today_key, current_submission_size)
instance.save()
client_record[today_key + "_size"] += current_submission_size
client_record.save()
return instance
