def generate_secure_text(self, origin_text: str) -> str:
"""
Encrypt and sign the origin text by ``cryptography`` library
using AES GCM algorithm.
:arg origin_text: is a string that will be encrypted by the provided
security secret.
"""
iv = os.urandom(16)
content = struct.pack(
"l", int(time.time())) + ensure_bytes(origin_text)
encryptor = AESCipher(
aes_algorithms.AES(ensure_bytes(self.__security_secret)),
aes_modes.GCM(iv),
backend=aes_backend()
).encryptor()
ciphertext = encryptor.update(content) + encryptor.finalize()
final_encrypted_text = iv
final_encrypted_text += struct.pack("l", len(ciphertext))
final_encrypted_text += ciphertext
final_encrypted_text += encryptor.tag
return ensure_str(base64.b64encode(final_encrypted_text))
def assemble(self) -> Tuple[bytes, str]:
"""
Generate HTTP v1 Body to bytes.
It will return the body in bytes and the content-type in str.
"""
body = b""
boundary = "----------FutureFinityFormBoundary"
boundary += ensure_str(security.get_random_str(8)).lower()
content_type = "multipart/form-data; boundary=" + boundary
full_boundary = b"--" + ensure_bytes(boundary)
for field_name, field_value in self.items():
body += full_boundary + _CRLF_BYTES_MARK
if isinstance(field_value, str):
body += b"Content-Disposition: form-data; "
body += ensure_bytes("name=\"%s\"\r\n" % field_name)
body += _CRLF_BYTES_MARK
body += ensure_bytes(field_value)
body += _CRLF_BYTES_MARK
else:
raise ProtocolError("Unknown Field Type")
for file_field in self.files.values():
body += full_boundary + _CRLF_BYTES_MARK
body += file_field.assemble()
body += full_boundary + b"--" + _CRLF_BYTES_MARK
return body, content_type
def check_etag_header(self) -> bool:
"""
Check etag header of response_body.
"""
computed_etag = ensure_bytes(self._body_etag)
etags = re.findall(
br'\*|(?:W/)?"[^"]*"',
ensure_bytes(self.get_header("if-none-match", ""))
)
if not computed_etag or not etags:
return False
match = False
if etags[0] == b'*':
match = True
else:
def value_validator(value):
if value.startswith(b'W/'):
value = value[2:]
return value
for etag in etags:
if value_validator(etag) == value_validator(computed_etag):
match = True
break
return match
async def post(
self, url: str,
headers: Union[protocol.HTTPHeaders, Mapping[str, str], None]=None,
cookies: Union[protocol.HTTPCookies, Mapping[str, str], None]=None,
link_args: Union[TolerantMagicDict, Mapping[str, str], None]=None,
body_args: Union[TolerantMagicDict, Mapping[str, str], None]=None,
files: Union[TolerantMagicDict, Mapping[str, str], None]=None):
"""
This is a friendly wrapper of `client.HTTPClient.fetch` for
`POST` request.
"""
if headers is None:
headers = protocol.HTTPHeaders()
else:
headers = protocol.HTTPHeaders(headers)
if "content-type" in headers.keys():
content_type = headers["content-type"]
if files:
if not content_type.lower().startswith("multipart/form-data"):
raise ClientError(
"Files can only be sent by multipart/form-data")
else:
if not files: # Automatic Content-Type Decision.
content_type = "application/x-www-form-urlencoded"
else:
content_type = "multipart/form-data"
if content_type.lower() == "application/x-www-form-urlencoded":
body = ensure_bytes(urllib.parse.urlencode(body_args))
elif content_type.lower() == "application/json":
body = ensure_bytes(json.dumps(body_args))
elif content_type.lower().startswith("multipart/form-data"):
multipart_body = protocol.HTTPMultipartBody()
multipart_body.update(body_args)
if files:
multipart_body.files.update(files)
body, content_type = multipart_body.assemble()
else:
raise ClientError("Unsupported Content-Type.")
content_length = str(len(body))
headers["content-length"] = content_length
headers["content-type"] = content_type
response = await self.fetch(method="POST", url=url, headers=headers,
cookies=cookies, link_args=link_args,
body=body)
return response
def assemble(self) -> bytes:
headers_str = ""
for (name, value) in self.items():
headers_str += "%s: %s" % (_capitalize_header[name], value)
headers_str += _CRLF_MARK
return ensure_bytes(headers_str)
def _parse_body(self):
if self.incoming._is_chunked_body:
self._parse_next_chunk()
return
if self._body_length is None:
self._body_length = self.incoming._expected_content_length
if self.is_client is True:
if self._body_length == -1:
return # Waiting For Connection Close.
if self._body_length > self.max_body_length:
raise ConnectionEntityTooLarge("The body is too large.")
if len(self._pending_bytes) < self._body_length:
return # Data not enough, waiting.
self._pending_body = ensure_bytes(
self._pending_bytes[:self._body_length])
del self._pending_bytes[:self._body_length]
self.incoming.body = self._pending_body
self.stage = _CONN_MESSAGE_PARSED
def test_aes_gcm_verify_data(self):
context = self.get_aes_gcm_context()
origin_text = futurefinity.security.get_random_str(100)
secure_text = context.generate_secure_text(origin_text)
failed_secure_text = bytearray(base64.b64decode(secure_text))
del failed_secure_text[-10:]
failed_secure_text += os.urandom(10)
failed_secure_text = base64.b64encode(ensure_bytes(failed_secure_text))
self.assertEqual(None, context.lookup_origin_text(failed_secure_text))
def __init__(self, security_secret: str):
if None in [AESCipher, aes_algorithms, aes_modes, aes_backend]:
raise FutureFinityError(
"Currently, `futurefinity.security.AESGCMSecurityObject` "
"needs Cryptography to work. Please install it before "
"using security features(such as security_secret), "
"or turn aes_security to False in Application Settings.")
self.__security_secret = hashlib.sha256(
ensure_bytes(security_secret)).digest()
def assemble(self) -> bytes:
"""
Assemble a HTTPHeaders Class to HTTP/1.x Form.
"""
headers_str = ""
for (name, value) in self.items():
headers_str += "%s: %s" % (_capitalize_header[name], value)
headers_str += _CRLF_MARK
return ensure_bytes(headers_str)
def generate_secure_text(self, origin_text: str) -> str:
"""
Generate a signed text by ``hmac`` library using sha256 algorithm.
:arg origin_text: is a string that will be signed by the provided
security secret.
"""
iv = os.urandom(16)
content = struct.pack(
"l", int(time.time())) + ensure_bytes(origin_text)
hash = hmac.new(iv + ensure_bytes(self.__security_secret),
digestmod=hashlib.sha256)
hash.update(ensure_bytes(content))
signature = hash.digest()
final_signed_text = iv
final_signed_text += struct.pack("l", len(content))
final_signed_text += content
final_signed_text += signature
return ensure_str(base64.b64encode(final_signed_text))
def lookup_origin_text(self, secure_text: str,
valid_length: int=None) -> str:
"""
Validate the signed text and return the origin text. If the content
and the signature mismatches, ``None`` will be returned.
:arg secure_text: is a base64 encoded string, which contains
the generated time, the content, and a sha256 signature.
:arg valid_length: the length that the content should be valid.
The unit is second. It you want the content be always valid, set
the length to ``None``.
"""
signed_text_reader = io.BytesIO(base64.b64decode(secure_text))
iv = signed_text_reader.read(16)
length = struct.unpack("l", signed_text_reader.read(8))[0]
content = signed_text_reader.read(length)
signature = signed_text_reader.read(32)
hash = hmac.new(iv + ensure_bytes(self.__security_secret),
digestmod=hashlib.sha256)
hash.update(ensure_bytes(content))
if not hmac.compare_digest(signature, hash.digest()):
return None
timestamp = struct.unpack("l", content[:8])[0]
text = content[8:]
if valid_length and int(time.time()) - timestamp > valid_length:
return None
try:
return ensure_str(text)
except:
return None
def write(self, text: typing.Union[str, bytes], clear_text: bool=False):
"""
Write response body.
If write() is called for many times, it will connect all text together.
If it is called after the request finished, it will raise an error.
"""
if self._finished:
raise HTTPError(
500, "Cannot write to request when it has already finished.")
self._body_written = True
if clear_text:
self._response_body.clear()
self._response_body += ensure_bytes(text)
def _parse_incoming_message(self):
self.controller.cancel_timeout_handler()
if self.is_client is False:
if self.stage is _CONN_INIT:
self.stage = _CONN_INITIAL_WAITING
if self.stage is _CONN_INITIAL_WAITING:
self._parse_initial()
if self.stage is _CONN_INITIAL_PARSED:
self.controller.initial_received(self.incoming)
if self.controller.use_stream:
self.stage = _CONN_STREAMED
elif not self.incoming._body_expected:
self.stage = _CONN_MESSAGE_PARSED
else:
self.stage = _CONN_BODY_WAITING
if not self.incoming._is_chunked_body:
if (self.incoming._expected_content_length == -1 and
not self.is_client):
raise ConnectionBadMessage(
"Method Request a body, "
"but we cannot find a way to detect body length.")
if self.stage is _CONN_STREAMED:
self.controller.stream_received(self.incoming,
ensure_bytes(self._pending_bytes))
self._pending_bytes.clear()
return
if self.stage is _CONN_BODY_WAITING:
self._parse_body()
if self.stage is _CONN_MESSAGE_PARSED:
self.controller.message_received(self.incoming)
if self.is_client:
if self._can_keep_alive:
self._reset_connection()
else:
self._close_connection()
return
def generate_secure_text(self, origin_text: str) -> str:
if not isinstance(origin_text, str):
raise TypeError("origin_text should be a string.")
iv = os.urandom(16)
content = struct.pack("l", int(time.time()))
content += ensure_bytes(origin_text)
hash = hmac.new(iv + self._security_secret, digestmod=hashlib.sha256)
hash.update(content)
signature = hash.digest()
final_signed_text = iv
final_signed_text += struct.pack("l", len(content))
final_signed_text += content
final_signed_text += signature
return ensure_str(base64.b64encode(final_signed_text))
def assemble(self) -> bytes:
"""
Convert this form field to bytes.
"""
self.headers["content-type"] = self.content_type
self.headers["content-transfer-encoding"] = self.encoding
content_disposition = "form-data; "
content_disposition += "name=\"%s\"; " % self.fieldname
content_disposition += "filename=\"%s\"" % self.filename
self.headers["content-disposition"] = content_disposition
field = self.headers.assemble()
field += _CRLF_BYTES_MARK
field += ensure_bytes(self.content)
field += _CRLF_BYTES_MARK
return field
def _write_body_chunk(self, body_chunk: bytes):
if not self._outgoing_chunked_body:
raise ProtocolError("Invalid Function Access.")
if not body_chunk:
return
# Prevent Body being finished accidentally.
# Finish Body Writing by HTTPv1Connection.finish_writing
chunk_bytes = b""
body_chunk_length = len(body_chunk)
chunk_bytes += ensure_bytes(hex(body_chunk_length)[2:].upper())
chunk_bytes += _CRLF_BYTES_MARK
chunk_bytes += body_chunk
chunk_bytes += _CRLF_BYTES_MARK
self.controller.transport.write(chunk_bytes)
def connection_lost(self, exc: typing.Optional[tuple]=None):
if self.stage is _CONN_CLOSED:
return # This connection has been closed.
if self.stage is _CONN_INIT:
self.stage = _CONN_CLOSED
return # This connection has nothing, so nothing to cleanup.
if self.is_client:
if self.stage is _CONN_BODY_WAITING:
self._pending_body = ensure_bytes(self._pending_bytes)
self._pending_bytes.clear()
self.incoming.body = self._pending_body
self.stage = _CONN_MESSAGE_PARSED
self._parse_incoming_message() # Trigger Message Received.
self._close_connection()
def lookup_origin_text(self, secure_text: str,
valid_length: int=None) -> str:
"""
Decrypt the encrypted text, validate the signature and
return the origin text. If the content and the signature mismatches,
``None`` will be returned.
:arg secure_text: is a base64 encoded string, which contains
the generated time, the content, and a AES GCM tag.
:arg valid_length: the length that the content should be valid.
The unit is second. It you want the content be always valid, set
the length to ``None``.
"""
encrypted_text_reader = io.BytesIO(base64.b64decode(secure_text))
iv = encrypted_text_reader.read(16)
length = struct.unpack("l", encrypted_text_reader.read(8))[0]
ciphertext = encrypted_text_reader.read(length)
tag = encrypted_text_reader.read(16)
decryptor = AESCipher(
aes_algorithms.AES(ensure_bytes(self.__security_secret)),
aes_modes.GCM(iv, tag),
backend=aes_backend()
).decryptor()
try:
content = decryptor.update(ciphertext) + decryptor.finalize()
except:
return None
timestamp = struct.unpack("l", content[:8])[0]
text = content[8:]
if valid_length and int(time.time()) - timestamp > valid_length:
return None
try:
return ensure_str(text)
except:
return None
def generate_secure_text(self, origin_text: str) -> str:
if not isinstance(origin_text, str):
raise TypeError("origin_text should be a string.")
iv = os.urandom(16)
content = struct.pack("l", int(time.time()))
content += ensure_bytes(origin_text)
encryptor = AESCipher(
aes_algorithms.AES(self._security_secret),
aes_modes.GCM(iv),
backend=aes_backend()
).encryptor()
ciphertext = encryptor.update(content) + encryptor.finalize()
final_encrypted_text = iv
final_encrypted_text += struct.pack("l", len(ciphertext))
final_encrypted_text += ciphertext
final_encrypted_text += encryptor.tag
return ensure_str(base64.b64encode(final_encrypted_text))
def test_ensure_bytes_from_none(self):
self.assertEqual(b"", ensure_bytes(None))
def _parse_initial(self):
initial_end = self._pending_bytes.find(_CRLF_BYTES_MARK * 2)
if initial_end == -1:
if len(self._pending_bytes) > self.max_initial_length:
raise ConnectionEntityTooLarge(
"Initial Exceed its Maximum Length.")
return
initial_end += 2
if initial_end > self.max_initial_length:
raise ConnectionEntityTooLarge(
"Initial Exceed its Maximum Length.")
return
pending_initial = ensure_bytes(self._pending_bytes[:initial_end])
del self._pending_bytes[:initial_end + 2]
basic_info, origin_headers = ensure_str(pending_initial).split(
_CRLF_MARK, 1)
basic_info = basic_info.split(" ")
if self.is_client:
http_version = basic_info[0]
if not basic_info[1].isdecimal():
raise ConnectionBadMessage("Bad Initial Received.")
self._parsed_incoming_info["status_code"] = int(basic_info[1])
else:
if len(basic_info) != 3:
raise ConnectionBadMessage("Bad Initial Received.")
method, origin_path, http_version = basic_info
self._parsed_incoming_info["method"] = basic_info[0]
self._parsed_incoming_info["origin_path"] = basic_info[1]
if http_version.lower() == "http/1.1":
self.http_version = 11
elif http_version.lower() == "http/1.0":
self.http_version = 10
else:
raise ConnectionBadMessage("Unknown HTTP Version.")
self._parsed_incoming_info["http_version"] = self.http_version
try:
headers = HTTPHeaders.parse(origin_headers)
except:
raise ConnectionBadMessage("Bad Headers Received.")
if self._can_keep_alive and "connection" in headers:
self._use_keep_alive = headers.get_first(
"connection").lower() == "keep-alive"
self._parsed_incoming_info["headers"] = headers
if self.is_client:
try:
self.incoming = HTTPIncomingResponse(
**self._parsed_incoming_info)
except:
raise ConnectionBadMessage("Bad Initial Received.")
else:
try:
self.incoming = HTTPIncomingRequest(
**self._parsed_incoming_info, connection=self)
except:
raise ConnectionBadMessage("Bad Initial Received.")
self.stage = _CONN_INITIAL_PARSED
def parse(content_type: str, data: bytes) -> "HTTPMultipartBody":
"""
Parse HTTP v1 Multipart Body.
It will raise an Error during the parse period if parse failed.
"""
body_args = HTTPMultipartBody()
if not content_type.lower().startswith("multipart/form-data"):
raise ProtocolError("Unknown content-type.")
for field in content_type.split(";"): # Search Boundary
if field.find("boundary=") == -1:
continue
boundary = ensure_bytes(field.split("=")[1])
if boundary.startswith(b'"') and boundary.endswith(b'"'):
boundary = boundary[1:-1]
break
else:
raise ProtocolError("Cannot Find Boundary.")
full_boundary = b"--" + boundary
body_content = data.split(full_boundary + b"--")[0]
full_boundary += _CRLF_BYTES_MARK
splitted_body_content = body_content.split(full_boundary)
for part in splitted_body_content:
if not part:
continue
initial, content = part.split(_CRLF_BYTES_MARK * 2)
headers = HTTPHeaders.parse(initial)
disposition = headers.get_first("content-disposition")
disposition_list = []
disposition_dict = TolerantMagicDict()
for field in disposition.split(";"): # Split Disposition
field = field.strip() # Remove Useless Spaces.
if field.find("=") == -1: # This is not a key-value pair.
disposition_list.append(field)
continue
key, value = field.split("=")
if value.startswith('"') and value.endswith('"'):
value = value[1:-1]
disposition_dict.add(key.strip().lower(), value.strip())
if disposition_list[0] != "form-data":
raise ProtocolError("Cannot Parse Body.")
# Mixed form-data will be supported later.
content = content[:-2] # Drop CRLF Mark
if "filename" in disposition_dict.keys():
body_args.files.add(
disposition_dict.get_first("name", ""),
HTTPMultipartFileField(
fieldname=disposition_dict.get_first("name", ""),
filename=disposition_dict.get_first("filename", ""),
content=content,
content_type=headers.get_first(
"content-type", "application/octet-stream"),
headers=headers,
encoding=headers.get_first("content-transfer-encoding",
"binary")))
else:
try:
content = content.decode()
except UnicodeDecodeError:
pass
body_args.add(disposition_dict.get_first("name", ""), content)
return body_args
def write_initial(
self, http_version: Optional[int]=None, method: str="GET",
path: str="/", status_code: int=200,
headers: Optional[HTTPHeaders]=None):
"""
Write the initial to remote.
"""
initial = b""
if http_version is not None:
self.http_version = http_version
if self.http_version == 11:
http_version_text = "HTTP/1.1"
elif self.http_version == 10:
http_version_text = "HTTP/1.0"
else:
raise ProtocolError("Unknown HTTP Version.")
basic_info_template = b"%s %s %s" + _CRLF_BYTES_MARK
if self.is_client:
if self.stage is not _CONN_INIT:
raise ProtocolError(
"Cannot write when connection stage is not _CONN_INIT.")
basic_info = basic_info_template % (
ensure_bytes(method), ensure_bytes(path),
ensure_bytes(http_version_text))
else:
if self.stage is not _CONN_MESSAGE_PARSED:
raise ProtocolError("Unacceptable Function Access.")
basic_info = basic_info_template % (
ensure_bytes(http_version_text), ensure_bytes(status_code),
ensure_bytes(status_code_text[status_code]))
initial += basic_info
if self._can_keep_alive and "connection" in headers:
self._use_keep_alive = headers.get_first(
"connection").lower() == "keep-alive"
transfer_encoding = headers.get_first("transfer-encoding")
if transfer_encoding is not None:
if transfer_encoding.lower() == "chunked":
self._outgoing_chunked_body = True
else:
self._outgoing_chunked_body = False
else:
self._outgoing_chunked_body = False
if "connection" not in headers.keys():
if self._can_keep_alive:
headers["connection"] = "Keep-Alive"
else:
headers["connection"] = "Close"
else:
self._use_keep_alive = headers[
"connection"].lower() == "keep-alive"
if self.is_client:
if "accept" not in headers.keys():
headers["accept"] = "*/*"
if "user-agent" not in headers.keys():
headers["user-agent"] = "futurefinity/" + futurefinity.version
else:
if "server" not in headers.keys():
headers["server"] = "futurefinity/" + futurefinity.version
if method.lower() == "head":
# For Head Request, there will not be a body.
self._outgoing_chunked_body = False
initial += headers.assemble()
initial += _CRLF_BYTES_MARK
self.controller.transport.write(initial)
self.stage = _CONN_INITIAL_WRITTEN
def __init__(self, security_secret: str):
self.__security_secret = hashlib.sha256(
ensure_bytes(security_secret)).digest()
def test_ensure_bytes_from_str(self):
random_str = futurefinity.security.get_random_str(10)
ensured_bytes = ensure_bytes(random_str)
self.assertEqual(random_str.encode(), ensured_bytes)
def test_ensure_bytes_from_other(self):
self.assertIsInstance(ensure_bytes(object()), bytes)
def test_ensure_bytes_from_bytes(self):
random_bytes = os.urandom(10)
ensured_bytes = ensure_bytes(random_bytes)
self.assertEqual(random_bytes, ensured_bytes)
def test_ensure_bytes_from_bytearray(self):
random_bytearray = bytearray(os.urandom(10))
ensured_bytes = ensure_bytes(random_bytearray)
self.assertEqual(bytes(random_bytearray), ensured_bytes)