def read(self, size=-1): """ Read up to size bytes from the object and return them. As a convenience, if size is unspecified or -1, all bytes until EOF are returned. If 0 bytes are returned, and size was not 0, this indicates end of file. :param int size: (optional) The number of bytes to attempt to read from the underlying stream. :rtype: bytes """ self._lock.acquire() try: if self._closed: raise ValueError("Cannot read from closed stream") if size == 0: return b"" self._handle_header() # indicates whether to read and return all content from underlying stream + header + (optional) tag READ_ALL = size <= -1 # we must read the entire ciphertext into memory to validate the tag before we # return any decrypted bytes # so if the buffer is not yet populated, read the entire stream into it if not self._finalized: self._finalized = True # read entire stream bytes_to_decrypt = convert_to_bytes(_read_full_stream_content(self._stream_to_decrypt)) tag = bytes_to_decrypt[-self._algorithm.tag_len:] bytes_to_decrypt = bytes_to_decrypt[: -self._algorithm.tag_len] decrypted_data = _update_cryptor_with_data( self._decryptor, bytes_to_decrypt ) decrypted_data += self._decryptor.finalize_with_tag(tag) self._buffer += decrypted_data # read output from buffer if READ_ALL: output = self._buffer[:] # remove the bytes that you just read from the buffer self._buffer = b"" else: output = self._buffer[:size] memview = memoryview(self._buffer) # remove the bytes that you just read from the buffer self._buffer = memview[size:] return output finally: self._lock.release()
def encrypt(**kwargs): """ Returns data encrypted under the provided master key. The master key is used to generate a data encryption key which is used directly to encrypt the data. The bytes returned in the CryptoResult include a header containing various metadata that allows it to be decrypted by the OCI Python SDK and other OCI SDKs that support client side encryption. Note this data cannot be decrypted using the KMS 'decrypt' APIs. :param oci.encryption.MasterKeyProvider master_key_provider: (required) A MasterKeyProvider to use for encrypting the data :param bytes data: (required) The data to be encyrpted. If a string is passed, it will be converted to bytes using UTF-8 encoding. Note that this conversion will require creating a copy of the data which may be undesirable for large payloads. :param dict encryption_context: (optional) The encryption context to use while encrypting the data. This must be a dict where all keys and values are strings, and no keys begin with the prefix "oci-". This context is used as additional authenticated data for authenticated encryption algorithms which support it. The same encryption context must be used upon decryption otherwise the call to decrypt will fail. The encryption context is included in the header of the encrypted payload, so you do not need to supply it separately upon decryption. :rtype: oci.encryption.CryptoResult """ _ensure_required_kwargs_present( required_kwargs=['master_key_provider', 'data'], provided_kwargs=kwargs) encryption_context = kwargs.get('encryption_context', None) # leaves input alone if it is alread bytes, otherwise converts to bytes using default encoding # this is for convenience of the caller, but will create a copy of the data if it is not already a # bytes-like object data = convert_to_bytes(kwargs.get('data')) # as long as we only read from the stream, BytesIO does not create a copy of the data so this doesn't # add memory overhead with io.BytesIO(data) as stream_to_encrypt: encryptor = StreamEncryptor( master_key_provider=kwargs.get('master_key_provider'), stream_to_encrypt=stream_to_encrypt, max_encryption_size=None, encryption_context=encryption_context, ) return CryptoResult(data=encryptor.read(), encryption_context=encryption_context)
def serialize_header(encrypted_data_header): encrypted_data_keys = [] for encrypted_data_key in encrypted_data_header.encrypted_data_keys: encrypted_data_keys.append({ ENCRYPTED_DATA_KEY_MASTER_KEY_ID: encrypted_data_key.master_key_id, ENCRYPTED_DATA_KEY_VAULT_ID: encrypted_data_key.vault_id, ENCRYPTED_DATA_KEY_ENCRYPTED_DATA_KEY: convert_bytes_to_base64_encoded_string( encrypted_data_key.encrypted_data_key_bytes), ENCRYPTED_DATA_KEY_REGION: encrypted_data_key.region, }) metadata = { METADATA_KEY_ENCRYPTED_CONTENT_FORMAT: encrypted_data_header.encrypted_content_format, METADATA_KEY_ENCRYPTED_DATA_KEYS: encrypted_data_keys, METADATA_KEY_IV: convert_bytes_to_base64_encoded_string(encrypted_data_header.iv_bytes), METADATA_KEY_ALGORITHM_ID: encrypted_data_header.algorithm_id, METADATA_KEY_ADDITIONAL_AUTHENTICATED_DATA: convert_to_str( encrypted_data_header.additional_authenticated_data_bytes), } json_header_as_string = json.dumps(metadata) header_format = STRUCT_HEADER_FORMAT.format( json_metadata_length=len(json_header_as_string)) packed_header = struct.pack( header_format, SERIALIZATION_FORMAT_VERSION, len(json_header_as_string), convert_to_bytes(json_header_as_string), ) return packed_header
def decrypt(**kwargs): """ Returns a CryptoResult containing decrypted bytes. This function requires that 'data' is in the format generated by the encrypt functionality in this SDK as well as other OCI SDKs that support client side encryption. Note this function cannot decrypt data encrypted by the KMS 'encrypt' APIs. :param oci.encryption.MasterKeyProvider master_key_provider: (required) A MasterKeyProvider to use for decrypting the data. :param bytes data: (required) The data to be decrypted. If a string is passed, it will be converted to bytes using UTF-8 encoding. Note that this conversion will require creating a copy of the data which may be undesirable for large payloads. :rtype: oci.encryption.CryptoResult """ _ensure_required_kwargs_present( required_kwargs=['master_key_provider', 'data'], provided_kwargs=kwargs) # leaves input alone if it is alread bytes, otherwise converts to bytes using default encoding # this is for convenience of the caller, but will create a copy of the data if it is not already a # bytes-like object data = convert_to_bytes(kwargs.get('data')) # as long as we only read from the stream, BytesIO does not create a copy of the data so this doesn't # add memory overhead with io.BytesIO(data) as stream_to_decrypt: decryptor = StreamDecryptor( stream_to_decrypt=stream_to_decrypt, master_key_provider=kwargs.get('master_key_provider')) return CryptoResult( data=decryptor.read(), encryption_context=decryptor.get_encryption_context())
def deserialize_header_from_stream(ciphertext_stream): short_format = ">H" short_size_offset = struct.calcsize(short_format) unsigned_int_format = ">I" unsigned_int_size_offset = struct.calcsize(unsigned_int_format) offset = 0 # get serialization format version next_content = ciphertext_stream.read(short_size_offset) (serialization_format_version, ) = struct.unpack_from( short_format, next_content, offset) offset = offset + short_size_offset if serialization_format_version != SERIALIZATION_FORMAT_VERSION: raise ValueError( "Could not deserialize header with unrecognized serialization format version: {}" .format(serialization_format_version)) # get json metadata length next_content = ciphertext_stream.read(unsigned_int_size_offset) (json_metadata_length, ) = struct.unpack_from(unsigned_int_format, next_content) offset = offset + short_size_offset # get json metadata chunk_format = "{}s".format(json_metadata_length) next_content = ciphertext_stream.read(struct.calcsize(chunk_format)) (json_metadata_bytes, ) = struct.unpack_from(chunk_format, next_content) offset = offset + struct.calcsize(chunk_format) json_metadata = convert_to_str(json_metadata_bytes) try: metadata = json.loads(json_metadata) except ValueError as e: raise ValueError( "Could not parse metadata inside header. Error: {}".format(str(e))) required_top_level_keys = [ METADATA_KEY_IV, METADATA_KEY_ALGORITHM_ID, METADATA_KEY_ADDITIONAL_AUTHENTICATED_DATA, ] required_encrypted_data_key_keys = [ ENCRYPTED_DATA_KEY_MASTER_KEY_ID, ENCRYPTED_DATA_KEY_VAULT_ID, ENCRYPTED_DATA_KEY_ENCRYPTED_DATA_KEY, ENCRYPTED_DATA_KEY_REGION, ] missing_or_none_top_level_keys = [ required_key for required_key in required_top_level_keys if (required_key not in metadata) or ( metadata.get(required_key, None) is None) or ( isinstance(metadata.get(required_key), list) and len(metadata.get(required_key)) == 0) ] if missing_or_none_top_level_keys: raise ValueError( "Invalid header. The following metadata keys must be present and not null: {}." .format(", ".join(missing_or_none_top_level_keys))) encrypted_data_keys_raw = metadata.get(METADATA_KEY_ENCRYPTED_DATA_KEYS) encrypted_data_keys = [] for encrypted_data_key_raw in encrypted_data_keys_raw: missing_or_none_dek_keys = [ required_key for required_key in required_encrypted_data_key_keys if (required_key not in encrypted_data_key_raw) or ( encrypted_data_key_raw.get(required_key, None) is None) ] if missing_or_none_dek_keys: raise ValueError( "Invalid header. The following metadata keys must be present and not null in each encrypted data key: {}." .format(", ".join(missing_or_none_dek_keys))) encrypted_data_keys.append( EncryptedDataHeaderDataEncryptionKey( master_key_id=encrypted_data_key_raw.get( ENCRYPTED_DATA_KEY_MASTER_KEY_ID), vault_id=encrypted_data_key_raw.get( ENCRYPTED_DATA_KEY_VAULT_ID), encrypted_data_key_bytes=convert_base64_encoded_string_to_bytes( encrypted_data_key_raw.get( ENCRYPTED_DATA_KEY_ENCRYPTED_DATA_KEY)), region=encrypted_data_key_raw.get(ENCRYPTED_DATA_KEY_REGION), )) header = EncryptedDataHeader( encrypted_data_keys=encrypted_data_keys, iv_bytes=convert_base64_encoded_string_to_bytes( metadata.get(METADATA_KEY_IV)), algorithm_id=metadata.get(METADATA_KEY_ALGORITHM_ID), additional_authenticated_data_bytes=convert_to_bytes( metadata.get(METADATA_KEY_ADDITIONAL_AUTHENTICATED_DATA)), ) return header
def _read_full_stream_content(stream): if hasattr(stream, 'readall'): return convert_to_bytes(stream.readall) else: return convert_to_bytes(stream.read())
def read(self, size=-1): """ Read up to size bytes from the object and return them. As a convenience, if size is unspecified or -1, all bytes until EOF are returned. If 0 bytes are returned, and size was not 0, this indicates end of file. :param int size: (optional) The number of bytes to attempt to read from the underlying stream. :rtype: bytes """ self._lock.acquire() try: if self._closed: raise ValueError("Cannot read from closed stream") if size == 0: return b"" READ_ALL = size <= -1 if not self._header_content: self._header_content = self._build_header_content() self._buffer += self._header_content if self._encryption_context_bytes: self.encryptor.authenticate_additional_data( self._encryption_context_bytes ) # try to throw early if this payload will exceed max encryption size self._enforce_max_encryption_size(size_of_next_read=size) # buffer doesn't have enough data to fulfill this read operation # so we need to load more info the buffer if not self._finalized and (size > len(self._buffer) or READ_ALL): if READ_ALL: size_to_read_into_buffer = -1 else: size_to_read_into_buffer = size - len(self._buffer) reached_end = False # read 'size_to_read_into_buffer' bytes from underlying stream if READ_ALL: bytes_to_encrypt = convert_to_bytes( self._stream_to_encrypt.read() ) else: bytes_to_encrypt = convert_to_bytes( self._stream_to_encrypt.read(size_to_read_into_buffer) ) total_bytes_to_encrypt_size = len(bytes_to_encrypt) ciphertext = _update_cryptor_with_data(self.encryptor, bytes_to_encrypt) reached_end = ( total_bytes_to_encrypt_size < size_to_read_into_buffer ) or READ_ALL if reached_end: ciphertext += self.encryptor.finalize() + self.encryptor.tag self._finalized = True self._buffer += ciphertext # read output from buffer if READ_ALL: output = self._buffer[:] # remove the bytes that you just read from the buffer self._buffer = b"" else: output = self._buffer[:size] # remove the bytes that you just read from the buffer self._buffer = self._buffer[size:] self._bytes_written_total += len(output) if self._bytes_written_total > len(self._header_content): self._bytes_written_excluding_header = self._bytes_written_total - len( self._header_content ) else: self._bytes_written_excluding_header = 0 # we try to determine the payload size ahead of time and throw before wasting time encrypting # in some cases we can't determine the stream length ahead of time so we check again before # returning to make sure we haven't encrypted too much data self._enforce_max_encryption_size(size_of_next_read=0) return output finally: self._lock.release()
def __init__( self, master_key_provider, stream_to_encrypt, max_encryption_size=DEFAULT_MAX_ENCRYPTION_SIZE_SENTINEL, encryption_context=None ): """ Provides a stream that wraps 'stream_to_encrypt' and allows reading data from the underlying stream encrypted under the provided master key. :param int max_encryption_size: (optional) Max number of bytes able to be encrypted by this StreamEncryptor. The default value differs based on the algorithm used. For GCM (the default algorithm) the default value is 2147483647 bytes. This is provided mainly for use with authenticated encryption algorithms that require verification of an authentication tag upon decryption. Because decrypting using these algorithms will buffer the entire payload into memory before returning it, this max_encryption_size provides a sanity check against encrypting payloads too large to decrypt. This is possible because encryption does not require holding the entire payload in memory. The 2147483647 byte limit was chosen because that is the maximum number of bytes that can be encrypted or decrypted by the OCI Java SDK. This is to avoid users accidentally encrypting payloads in Python that cannot be decrypted in Java. Explicitly passing this value as None will disable the size check and allow encrypting payloads up to the maximum size supported by the algorithm. :param dict encryption_context: (optional) Optional additional data to be provided as input to authenticated encryption algorithms. This must be a dict with keys that are strings and values that are strings. Keys may NOT match the prefix oci-* as that namespace is reserved for OCI internal keys that may be added to the AAD. """ self._algorithm = DEFAULT_ALGORITHM self._primary_master_key = master_key_provider.get_primary_master_key() if not self._primary_master_key: raise ValueError("master_key_provider must contain a primary master key in order to encrypt data") self._data_encryption_key = self._primary_master_key.generate_data_encryption_key( self._algorithm ) self._stream_to_encrypt = stream_to_encrypt _validate_encryption_context(encryption_context) self._encryption_context = encryption_context self._encryption_context_bytes = convert_to_bytes( convert_encryption_context_to_string(encryption_context) ) self.iv = generate_random_iv(self._algorithm.iv_len) cipher = Cipher( algorithm=self._algorithm.algorithm( self._data_encryption_key.plaintext_key_bytes ), mode=self._algorithm.mode(self.iv), backend=default_backend(), ) # use encryptor directly instead of AESGCM class # https://cryptography.io/en/latest/hazmat/primitives/aead/#cryptography.hazmat.primitives.ciphers.aead.AESGCM # so that we can optionally stream data during encryption without holding the entire payload in memory self.encryptor = cipher.encryptor() self._max_encryption_size = None if max_encryption_size is DEFAULT_MAX_ENCRYPTION_SIZE_SENTINEL: if self._algorithm.mode.name == modes.GCM.name: self._max_encryption_size = DEFAULT_MAX_GCM_ENCRYPTION_SIZE else: raise ValueError("Unrecognized algorithm provided: {}".format(str(self._algorithm))) elif max_encryption_size is not None: if not isinstance(max_encryption_size, int): raise TypeError("argument max_encryption_size must be an integer") self._max_encryption_size = max_encryption_size self._header_content = None self._buffer = b"" self._bytes_written_total = 0 self._bytes_written_excluding_header = 0 self._finalized = False self._lock = Lock() self._closed = False