def test_encrypted_upload_wal(self, rfo_mock, boto_mock):
"""
Test the upload of a WAL
"""
# Create a simple CloudWalUploader obj
cloud_interface = S3CloudInterface("s3://bucket/path/to/dir",
encryption="AES256")
uploader = CloudWalUploader(cloud_interface, "test-server")
source = "/wal_dir/000000080000ABFF000000C1"
# Simulate the file object returned by the retrieve_file_obj method
rfo_mock.return_value.name = source
uploader.upload_wal(source)
session_mock = boto_mock.Session.return_value
s3_client_mock = session_mock.resource.return_value.meta.client
# Check the call for the creation of the destination key
s3_client_mock.upload_fileobj.assert_called_once_with(
Fileobj=rfo_mock.return_value,
Bucket=cloud_interface.bucket_name,
Key=os.path.join(
cloud_interface.path,
uploader.server_name,
"wals",
hash_dir(source),
os.path.basename(source),
),
ExtraArgs={"ServerSideEncryption": "AES256"},
)
def test_upload_wal(self, rfo_mock, ContainerClientMock):
"""
Test the upload of a WAL
"""
# Create a simple S3WalUploader obj
container_name = "container"
cloud_interface = AzureCloudInterface(
url="https://account.blob.core.windows.net/container/path/to/dir")
uploader = CloudWalUploader(cloud_interface, "test-server")
source = "/wal_dir/000000080000ABFF000000C1"
# Simulate the file object returned by the retrieve_file_obj method
rfo_mock.return_value.name = source
uploader.upload_wal(source)
ContainerClientMock.from_connection_string.assert_called_once_with(
conn_str=os.environ["AZURE_STORAGE_CONNECTION_STRING"],
container_name=container_name,
)
container_client = ContainerClientMock.from_connection_string.return_value
# Check the call for the creation of the destination key
container_client.upload_blob.assert_called_once_with(
data=rfo_mock.return_value,
name=os.path.join(
cloud_interface.path,
uploader.server_name,
"wals",
hash_dir(source),
os.path.basename(source),
),
overwrite=True,
)
def test_retrieve_normal_file_name(self):
"""
Test the retrieve_wal_name method with an uncompressed file
"""
# Create a fake source name
source = "wal_dir/000000080000ABFF000000C1"
uploader = CloudWalUploader(mock.MagicMock(), "test-server")
wal_final_name = uploader.retrieve_wal_name(source)
# Check the file name received
assert wal_final_name
assert wal_final_name == "000000080000ABFF000000C1"
def test_retrieve_normal_file_obj(self, tmpdir):
"""
Test the retrieve_file_obj method with an uncompressed file
"""
# Setup the WAL file
source = tmpdir.join("wal_dir/000000080000ABFF000000C1")
source.write("something".encode("utf-8"), ensure=True)
# Create a simple CloudWalUploader obj
uploader = CloudWalUploader(mock.MagicMock(), "test-server")
open_file = uploader.retrieve_file_obj(source.strpath)
# Check the file received
assert open_file
# Check content
assert open_file.read() == "something".encode("utf-8")
def test_retrieve_bz2_file_obj(self, tmpdir):
"""
Test the retrieve_file_obj method with a bz2 file
"""
# Setup the WAL
source = tmpdir.join("wal_dir/000000080000ABFF000000C1")
source.write("something".encode("utf-8"), ensure=True)
# Create a simple CloudWalUploader obj
uploader = CloudWalUploader(mock.MagicMock(),
"test-server",
compression="bzip2")
open_file = uploader.retrieve_file_obj(source.strpath)
# Check the in memory file received
assert open_file
# Decompress on the fly to check content
assert bz2.decompress(open_file.read()) == "something".encode("utf-8")
def test_upload_wal(self, rfo_mock, cloud_interface_mock):
"""
Test upload_wal calls CloudInterface with expected parameters
"""
bucket_path = "gs://bucket/path/to/dir"
server_name = "test_server"
type(cloud_interface_mock).path = mock.PropertyMock(
return_value=bucket_path)
uploader = CloudWalUploader(cloud_interface_mock, server_name)
source = "/wal_dir/000000080000ABFF000000C1"
# Simulate the file object returned by the retrieve_file_obj method
rfo_mock.return_value.name = source
mock_fileobj_length = 42
rfo_mock.return_value.tell.return_value = mock_fileobj_length
uploader.upload_wal(source)
expected_key = os.path.join(bucket_path, server_name, "wals",
hash_dir(source), os.path.basename(source))
cloud_interface_mock.upload_fileobj.assert_called_once_with(
fileobj=rfo_mock(), key=expected_key, override_tags=None)