def test_encrypted_upload_wal(self, rfo_mock, boto_mock): """ Test the upload of a WAL """ # Create a simple CloudWalUploader obj cloud_interface = S3CloudInterface("s3://bucket/path/to/dir", encryption="AES256") uploader = CloudWalUploader(cloud_interface, "test-server") source = "/wal_dir/000000080000ABFF000000C1" # Simulate the file object returned by the retrieve_file_obj method rfo_mock.return_value.name = source uploader.upload_wal(source) session_mock = boto_mock.Session.return_value s3_client_mock = session_mock.resource.return_value.meta.client # Check the call for the creation of the destination key s3_client_mock.upload_fileobj.assert_called_once_with( Fileobj=rfo_mock.return_value, Bucket=cloud_interface.bucket_name, Key=os.path.join( cloud_interface.path, uploader.server_name, "wals", hash_dir(source), os.path.basename(source), ), ExtraArgs={"ServerSideEncryption": "AES256"}, )
def test_upload_wal(self, rfo_mock, ContainerClientMock): """ Test the upload of a WAL """ # Create a simple S3WalUploader obj container_name = "container" cloud_interface = AzureCloudInterface( url="https://account.blob.core.windows.net/container/path/to/dir") uploader = CloudWalUploader(cloud_interface, "test-server") source = "/wal_dir/000000080000ABFF000000C1" # Simulate the file object returned by the retrieve_file_obj method rfo_mock.return_value.name = source uploader.upload_wal(source) ContainerClientMock.from_connection_string.assert_called_once_with( conn_str=os.environ["AZURE_STORAGE_CONNECTION_STRING"], container_name=container_name, ) container_client = ContainerClientMock.from_connection_string.return_value # Check the call for the creation of the destination key container_client.upload_blob.assert_called_once_with( data=rfo_mock.return_value, name=os.path.join( cloud_interface.path, uploader.server_name, "wals", hash_dir(source), os.path.basename(source), ), overwrite=True, )
def test_retrieve_normal_file_name(self): """ Test the retrieve_wal_name method with an uncompressed file """ # Create a fake source name source = "wal_dir/000000080000ABFF000000C1" uploader = CloudWalUploader(mock.MagicMock(), "test-server") wal_final_name = uploader.retrieve_wal_name(source) # Check the file name received assert wal_final_name assert wal_final_name == "000000080000ABFF000000C1"
def test_retrieve_normal_file_obj(self, tmpdir): """ Test the retrieve_file_obj method with an uncompressed file """ # Setup the WAL file source = tmpdir.join("wal_dir/000000080000ABFF000000C1") source.write("something".encode("utf-8"), ensure=True) # Create a simple CloudWalUploader obj uploader = CloudWalUploader(mock.MagicMock(), "test-server") open_file = uploader.retrieve_file_obj(source.strpath) # Check the file received assert open_file # Check content assert open_file.read() == "something".encode("utf-8")
def test_retrieve_bz2_file_obj(self, tmpdir): """ Test the retrieve_file_obj method with a bz2 file """ # Setup the WAL source = tmpdir.join("wal_dir/000000080000ABFF000000C1") source.write("something".encode("utf-8"), ensure=True) # Create a simple CloudWalUploader obj uploader = CloudWalUploader(mock.MagicMock(), "test-server", compression="bzip2") open_file = uploader.retrieve_file_obj(source.strpath) # Check the in memory file received assert open_file # Decompress on the fly to check content assert bz2.decompress(open_file.read()) == "something".encode("utf-8")
def test_upload_wal(self, rfo_mock, cloud_interface_mock): """ Test upload_wal calls CloudInterface with expected parameters """ bucket_path = "gs://bucket/path/to/dir" server_name = "test_server" type(cloud_interface_mock).path = mock.PropertyMock( return_value=bucket_path) uploader = CloudWalUploader(cloud_interface_mock, server_name) source = "/wal_dir/000000080000ABFF000000C1" # Simulate the file object returned by the retrieve_file_obj method rfo_mock.return_value.name = source mock_fileobj_length = 42 rfo_mock.return_value.tell.return_value = mock_fileobj_length uploader.upload_wal(source) expected_key = os.path.join(bucket_path, server_name, "wals", hash_dir(source), os.path.basename(source)) cloud_interface_mock.upload_fileobj.assert_called_once_with( fileobj=rfo_mock(), key=expected_key, override_tags=None)