def read(self, size=-1):
"""
Read up to size bytes from the object and return them. As a convenience, if size is unspecified or -1,
all bytes until EOF are returned.
If 0 bytes are returned, and size was not 0, this indicates end of file.
:param int size: (optional)
The number of bytes to attempt to read from the underlying stream.
:rtype: bytes
"""
self._lock.acquire()
try:
if self._closed:
raise ValueError("Cannot read from closed stream")
if size == 0:
return b""
self._handle_header()
# indicates whether to read and return all content from underlying stream + header + (optional) tag
READ_ALL = size <= -1
# we must read the entire ciphertext into memory to validate the tag before we
# return any decrypted bytes
# so if the buffer is not yet populated, read the entire stream into it
if not self._finalized:
self._finalized = True
# read entire stream
bytes_to_decrypt = convert_to_bytes(_read_full_stream_content(self._stream_to_decrypt))
tag = bytes_to_decrypt[-self._algorithm.tag_len:]
bytes_to_decrypt = bytes_to_decrypt[: -self._algorithm.tag_len]
decrypted_data = _update_cryptor_with_data(
self._decryptor, bytes_to_decrypt
)
decrypted_data += self._decryptor.finalize_with_tag(tag)
self._buffer += decrypted_data
# read output from buffer
if READ_ALL:
output = self._buffer[:]
# remove the bytes that you just read from the buffer
self._buffer = b""
else:
output = self._buffer[:size]
memview = memoryview(self._buffer)
# remove the bytes that you just read from the buffer
self._buffer = memview[size:]
return output
finally:
self._lock.release()
def encrypt(**kwargs):
"""
Returns data encrypted under the provided master key.
The master key is used to generate a data encryption key which
is used directly to encrypt the data.
The bytes returned in the CryptoResult include a header containing various
metadata that allows it to be decrypted by the OCI Python SDK and other OCI
SDKs that support client side encryption.
Note this data cannot be decrypted using the KMS 'decrypt' APIs.
:param oci.encryption.MasterKeyProvider master_key_provider: (required)
A MasterKeyProvider to use for encrypting the data
:param bytes data: (required)
The data to be encyrpted. If a string is passed, it will be converted to
bytes using UTF-8 encoding. Note that this conversion will require creating
a copy of the data which may be undesirable for large payloads.
:param dict encryption_context: (optional)
The encryption context to use while encrypting the data. This must be a dict where
all keys and values are strings, and no keys begin with the prefix "oci-".
This context is used as additional authenticated data for authenticated encryption
algorithms which support it. The same encryption context must be used upon decryption
otherwise the call to decrypt will fail. The encryption context is included in the
header of the encrypted payload, so you do not need to supply it separately upon
decryption.
:rtype: oci.encryption.CryptoResult
"""
_ensure_required_kwargs_present(
required_kwargs=['master_key_provider', 'data'],
provided_kwargs=kwargs)
encryption_context = kwargs.get('encryption_context', None)
# leaves input alone if it is alread bytes, otherwise converts to bytes using default encoding
# this is for convenience of the caller, but will create a copy of the data if it is not already a
# bytes-like object
data = convert_to_bytes(kwargs.get('data'))
# as long as we only read from the stream, BytesIO does not create a copy of the data so this doesn't
# add memory overhead
with io.BytesIO(data) as stream_to_encrypt:
encryptor = StreamEncryptor(
master_key_provider=kwargs.get('master_key_provider'),
stream_to_encrypt=stream_to_encrypt,
max_encryption_size=None,
encryption_context=encryption_context,
)
return CryptoResult(data=encryptor.read(),
encryption_context=encryption_context)
def serialize_header(encrypted_data_header):
encrypted_data_keys = []
for encrypted_data_key in encrypted_data_header.encrypted_data_keys:
encrypted_data_keys.append({
ENCRYPTED_DATA_KEY_MASTER_KEY_ID:
encrypted_data_key.master_key_id,
ENCRYPTED_DATA_KEY_VAULT_ID:
encrypted_data_key.vault_id,
ENCRYPTED_DATA_KEY_ENCRYPTED_DATA_KEY:
convert_bytes_to_base64_encoded_string(
encrypted_data_key.encrypted_data_key_bytes),
ENCRYPTED_DATA_KEY_REGION:
encrypted_data_key.region,
})
metadata = {
METADATA_KEY_ENCRYPTED_CONTENT_FORMAT:
encrypted_data_header.encrypted_content_format,
METADATA_KEY_ENCRYPTED_DATA_KEYS:
encrypted_data_keys,
METADATA_KEY_IV:
convert_bytes_to_base64_encoded_string(encrypted_data_header.iv_bytes),
METADATA_KEY_ALGORITHM_ID:
encrypted_data_header.algorithm_id,
METADATA_KEY_ADDITIONAL_AUTHENTICATED_DATA:
convert_to_str(
encrypted_data_header.additional_authenticated_data_bytes),
}
json_header_as_string = json.dumps(metadata)
header_format = STRUCT_HEADER_FORMAT.format(
json_metadata_length=len(json_header_as_string))
packed_header = struct.pack(
header_format,
SERIALIZATION_FORMAT_VERSION,
len(json_header_as_string),
convert_to_bytes(json_header_as_string),
)
return packed_header
def decrypt(**kwargs):
"""
Returns a CryptoResult containing decrypted bytes.
This function requires that 'data' is in the format generated by the
encrypt functionality in this SDK as well as other OCI SDKs that support
client side encryption.
Note this function cannot decrypt data encrypted by the KMS 'encrypt' APIs.
:param oci.encryption.MasterKeyProvider master_key_provider: (required)
A MasterKeyProvider to use for decrypting the data.
:param bytes data: (required)
The data to be decrypted. If a string is passed, it will be converted to
bytes using UTF-8 encoding. Note that this conversion will require creating
a copy of the data which may be undesirable for large payloads.
:rtype: oci.encryption.CryptoResult
"""
_ensure_required_kwargs_present(
required_kwargs=['master_key_provider', 'data'],
provided_kwargs=kwargs)
# leaves input alone if it is alread bytes, otherwise converts to bytes using default encoding
# this is for convenience of the caller, but will create a copy of the data if it is not already a
# bytes-like object
data = convert_to_bytes(kwargs.get('data'))
# as long as we only read from the stream, BytesIO does not create a copy of the data so this doesn't
# add memory overhead
with io.BytesIO(data) as stream_to_decrypt:
decryptor = StreamDecryptor(
stream_to_decrypt=stream_to_decrypt,
master_key_provider=kwargs.get('master_key_provider'))
return CryptoResult(
data=decryptor.read(),
encryption_context=decryptor.get_encryption_context())
def deserialize_header_from_stream(ciphertext_stream):
short_format = ">H"
short_size_offset = struct.calcsize(short_format)
unsigned_int_format = ">I"
unsigned_int_size_offset = struct.calcsize(unsigned_int_format)
offset = 0
# get serialization format version
next_content = ciphertext_stream.read(short_size_offset)
(serialization_format_version, ) = struct.unpack_from(
short_format, next_content, offset)
offset = offset + short_size_offset
if serialization_format_version != SERIALIZATION_FORMAT_VERSION:
raise ValueError(
"Could not deserialize header with unrecognized serialization format version: {}"
.format(serialization_format_version))
# get json metadata length
next_content = ciphertext_stream.read(unsigned_int_size_offset)
(json_metadata_length, ) = struct.unpack_from(unsigned_int_format,
next_content)
offset = offset + short_size_offset
# get json metadata
chunk_format = "{}s".format(json_metadata_length)
next_content = ciphertext_stream.read(struct.calcsize(chunk_format))
(json_metadata_bytes, ) = struct.unpack_from(chunk_format, next_content)
offset = offset + struct.calcsize(chunk_format)
json_metadata = convert_to_str(json_metadata_bytes)
try:
metadata = json.loads(json_metadata)
except ValueError as e:
raise ValueError(
"Could not parse metadata inside header. Error: {}".format(str(e)))
required_top_level_keys = [
METADATA_KEY_IV,
METADATA_KEY_ALGORITHM_ID,
METADATA_KEY_ADDITIONAL_AUTHENTICATED_DATA,
]
required_encrypted_data_key_keys = [
ENCRYPTED_DATA_KEY_MASTER_KEY_ID,
ENCRYPTED_DATA_KEY_VAULT_ID,
ENCRYPTED_DATA_KEY_ENCRYPTED_DATA_KEY,
ENCRYPTED_DATA_KEY_REGION,
]
missing_or_none_top_level_keys = [
required_key for required_key in required_top_level_keys
if (required_key not in metadata) or (
metadata.get(required_key, None) is None) or (
isinstance(metadata.get(required_key), list)
and len(metadata.get(required_key)) == 0)
]
if missing_or_none_top_level_keys:
raise ValueError(
"Invalid header. The following metadata keys must be present and not null: {}."
.format(", ".join(missing_or_none_top_level_keys)))
encrypted_data_keys_raw = metadata.get(METADATA_KEY_ENCRYPTED_DATA_KEYS)
encrypted_data_keys = []
for encrypted_data_key_raw in encrypted_data_keys_raw:
missing_or_none_dek_keys = [
required_key for required_key in required_encrypted_data_key_keys
if (required_key not in encrypted_data_key_raw) or (
encrypted_data_key_raw.get(required_key, None) is None)
]
if missing_or_none_dek_keys:
raise ValueError(
"Invalid header. The following metadata keys must be present and not null in each encrypted data key: {}."
.format(", ".join(missing_or_none_dek_keys)))
encrypted_data_keys.append(
EncryptedDataHeaderDataEncryptionKey(
master_key_id=encrypted_data_key_raw.get(
ENCRYPTED_DATA_KEY_MASTER_KEY_ID),
vault_id=encrypted_data_key_raw.get(
ENCRYPTED_DATA_KEY_VAULT_ID),
encrypted_data_key_bytes=convert_base64_encoded_string_to_bytes(
encrypted_data_key_raw.get(
ENCRYPTED_DATA_KEY_ENCRYPTED_DATA_KEY)),
region=encrypted_data_key_raw.get(ENCRYPTED_DATA_KEY_REGION),
))
header = EncryptedDataHeader(
encrypted_data_keys=encrypted_data_keys,
iv_bytes=convert_base64_encoded_string_to_bytes(
metadata.get(METADATA_KEY_IV)),
algorithm_id=metadata.get(METADATA_KEY_ALGORITHM_ID),
additional_authenticated_data_bytes=convert_to_bytes(
metadata.get(METADATA_KEY_ADDITIONAL_AUTHENTICATED_DATA)),
)
return header
def _read_full_stream_content(stream):
if hasattr(stream, 'readall'):
return convert_to_bytes(stream.readall)
else:
return convert_to_bytes(stream.read())
def read(self, size=-1):
"""
Read up to size bytes from the object and return them. As a convenience, if size is unspecified or -1,
all bytes until EOF are returned.
If 0 bytes are returned, and size was not 0, this indicates end of file.
:param int size: (optional)
The number of bytes to attempt to read from the underlying stream.
:rtype: bytes
"""
self._lock.acquire()
try:
if self._closed:
raise ValueError("Cannot read from closed stream")
if size == 0:
return b""
READ_ALL = size <= -1
if not self._header_content:
self._header_content = self._build_header_content()
self._buffer += self._header_content
if self._encryption_context_bytes:
self.encryptor.authenticate_additional_data(
self._encryption_context_bytes
)
# try to throw early if this payload will exceed max encryption size
self._enforce_max_encryption_size(size_of_next_read=size)
# buffer doesn't have enough data to fulfill this read operation
# so we need to load more info the buffer
if not self._finalized and (size > len(self._buffer) or READ_ALL):
if READ_ALL:
size_to_read_into_buffer = -1
else:
size_to_read_into_buffer = size - len(self._buffer)
reached_end = False
# read 'size_to_read_into_buffer' bytes from underlying stream
if READ_ALL:
bytes_to_encrypt = convert_to_bytes(
self._stream_to_encrypt.read()
)
else:
bytes_to_encrypt = convert_to_bytes(
self._stream_to_encrypt.read(size_to_read_into_buffer)
)
total_bytes_to_encrypt_size = len(bytes_to_encrypt)
ciphertext = _update_cryptor_with_data(self.encryptor, bytes_to_encrypt)
reached_end = (
total_bytes_to_encrypt_size < size_to_read_into_buffer
) or READ_ALL
if reached_end:
ciphertext += self.encryptor.finalize() + self.encryptor.tag
self._finalized = True
self._buffer += ciphertext
# read output from buffer
if READ_ALL:
output = self._buffer[:]
# remove the bytes that you just read from the buffer
self._buffer = b""
else:
output = self._buffer[:size]
# remove the bytes that you just read from the buffer
self._buffer = self._buffer[size:]
self._bytes_written_total += len(output)
if self._bytes_written_total > len(self._header_content):
self._bytes_written_excluding_header = self._bytes_written_total - len(
self._header_content
)
else:
self._bytes_written_excluding_header = 0
# we try to determine the payload size ahead of time and throw before wasting time encrypting
# in some cases we can't determine the stream length ahead of time so we check again before
# returning to make sure we haven't encrypted too much data
self._enforce_max_encryption_size(size_of_next_read=0)
return output
finally:
self._lock.release()
def __init__(
self,
master_key_provider,
stream_to_encrypt,
max_encryption_size=DEFAULT_MAX_ENCRYPTION_SIZE_SENTINEL,
encryption_context=None
):
"""
Provides a stream that wraps 'stream_to_encrypt' and allows reading data from the underlying stream
encrypted under the provided master key.
:param int max_encryption_size: (optional)
Max number of bytes able to be encrypted by this StreamEncryptor. The default value differs
based on the algorithm used. For GCM (the default algorithm) the default value is 2147483647 bytes.
This is provided mainly for use with authenticated encryption algorithms that require verification
of an authentication tag upon decryption. Because decrypting using these algorithms will buffer
the entire payload into memory before returning it, this max_encryption_size provides a sanity
check against encrypting payloads too large to decrypt. This is possible because encryption does not
require holding the entire payload in memory.
The 2147483647 byte limit was chosen because that is the maximum number of bytes that can be encrypted or
decrypted by the OCI Java SDK. This is to avoid users accidentally encrypting payloads in Python that
cannot be decrypted in Java.
Explicitly passing this value as None will disable the size check and allow encrypting payloads up to
the maximum size supported by the algorithm.
:param dict encryption_context: (optional)
Optional additional data to be provided as input to authenticated encryption
algorithms. This must be a dict with keys that are strings and values that are strings. Keys may NOT
match the prefix oci-* as that namespace is reserved for OCI internal keys that may be added to the AAD.
"""
self._algorithm = DEFAULT_ALGORITHM
self._primary_master_key = master_key_provider.get_primary_master_key()
if not self._primary_master_key:
raise ValueError("master_key_provider must contain a primary master key in order to encrypt data")
self._data_encryption_key = self._primary_master_key.generate_data_encryption_key(
self._algorithm
)
self._stream_to_encrypt = stream_to_encrypt
_validate_encryption_context(encryption_context)
self._encryption_context = encryption_context
self._encryption_context_bytes = convert_to_bytes(
convert_encryption_context_to_string(encryption_context)
)
self.iv = generate_random_iv(self._algorithm.iv_len)
cipher = Cipher(
algorithm=self._algorithm.algorithm(
self._data_encryption_key.plaintext_key_bytes
),
mode=self._algorithm.mode(self.iv),
backend=default_backend(),
)
# use encryptor directly instead of AESGCM class
# https://cryptography.io/en/latest/hazmat/primitives/aead/#cryptography.hazmat.primitives.ciphers.aead.AESGCM
# so that we can optionally stream data during encryption without holding the entire payload in memory
self.encryptor = cipher.encryptor()
self._max_encryption_size = None
if max_encryption_size is DEFAULT_MAX_ENCRYPTION_SIZE_SENTINEL:
if self._algorithm.mode.name == modes.GCM.name:
self._max_encryption_size = DEFAULT_MAX_GCM_ENCRYPTION_SIZE
else:
raise ValueError("Unrecognized algorithm provided: {}".format(str(self._algorithm)))
elif max_encryption_size is not None:
if not isinstance(max_encryption_size, int):
raise TypeError("argument max_encryption_size must be an integer")
self._max_encryption_size = max_encryption_size
self._header_content = None
self._buffer = b""
self._bytes_written_total = 0
self._bytes_written_excluding_header = 0
self._finalized = False
self._lock = Lock()
self._closed = False
