async def test_list_blobs(self, resource_group, location, storage_account,
storage_account_key):
# Arrange
bsc = BlobServiceClient(self.account_url(storage_account, "blob"),
credential=storage_account_key,
connection_data_block_size=1024,
max_single_put_size=1024,
min_large_block_upload_threshold=1024,
max_block_size=1024,
max_page_size=1024)
await self._setup(bsc)
blob_client, _ = await self._create_block_blob(
bsc,
blob_name="blockblob",
data=b'AAABBBCCC',
encryption_scope=TEST_ENCRYPTION_KEY_SCOPE)
await self._create_append_blob(
bsc, encryption_scope=TEST_ENCRYPTION_KEY_SCOPE)
container_client = bsc.get_container_client(self.container_name)
generator = container_client.list_blobs(include="metadata")
async for blob in generator:
self.assertIsNotNone(blob)
# Assert: every listed blob has encryption_scope
self.assertEqual(blob.encryption_scope, TEST_ENCRYPTION_KEY_SCOPE)
self._teardown(bsc)
async def return_gallery_blobs(start=0, limit=None):
blob_service_client = BlobServiceClient(
account_url=f"https://{os.getenv('AZURE_STORAGE_ACCOUNT')}.blob.core.windows.net/",
credential=os.getenv("AZURE_STORAGE_KEY"),
)
container_client = blob_service_client.get_container_client(
os.getenv("AZURE_STORAGE_VIDEO_CONTAINER")
)
blobs_list = []
async for blob in container_client.list_blobs( # pylint: disable=E1133
include=["metadata"]
):
metadata = blob.metadata
created_at = blob.creation_time
blobs_list.append(
{
"uuid": metadata["uuid"],
"image_url": f"/get_thumbnail?video_uuid={metadata['uuid']}",
"uploader_username": metadata["uploader_username"],
"uploader_id": metadata["uploader_id"],
"title": metadata["title"],
"badge": metadata["badge"],
"elapsed_time": date_funcs.elapsed_time_str(created_at),
}
)
await container_client.close()
await blob_service_client.close()
return blobs_list
def test_service_client_api_version_property(self):
service_client = BlobServiceClient(
"https://foo.blob.core.windows.net/account", credential="fake_key")
self.assertEqual(service_client.api_version, self.api_version_2)
self.assertEqual(service_client._client._config.version,
self.api_version_2)
with pytest.raises(AttributeError):
service_client.api_version = "foo"
service_client = BlobServiceClient(
"https://foo.blob.core.windows.net/account",
credential="fake_key",
api_version=self.api_version_1)
self.assertEqual(service_client.api_version, self.api_version_1)
self.assertEqual(service_client._client._config.version,
self.api_version_1)
container_client = service_client.get_container_client("foo")
self.assertEqual(container_client.api_version, self.api_version_1)
self.assertEqual(container_client._client._config.version,
self.api_version_1)
blob_client = service_client.get_blob_client("foo", "bar")
self.assertEqual(blob_client.api_version, self.api_version_1)
self.assertEqual(blob_client._client._config.version,
self.api_version_1)
async def test_sas_signature_is_scrubbed_off(self, storage_account_name,
storage_account_key):
# Test can only run live
bsc = BlobServiceClient(self.account_url(storage_account_name, "blob"),
storage_account_key)
await self._setup(bsc)
# Arrange
container = bsc.get_container_client(self.container_name)
token = generate_container_sas(
container.account_name,
container.container_name,
account_key=container.credential.account_key,
permission=ContainerSasPermissions(read=True),
expiry=datetime.utcnow() + timedelta(hours=1),
)
# parse out the signed signature
token_components = parse_qs(token)
signed_signature = quote(
token_components[QueryStringConstants.SIGNED_SIGNATURE][0])
sas_service = ContainerClient.from_container_url(container.url,
credential=token)
# Act
with LogCaptured(self) as log_captured:
await sas_service.get_account_information(logging_enable=True)
log_as_str = log_captured.getvalue()
# Assert
# make sure the query parameter 'sig' is logged, but its value is not
self.assertTrue(
QueryStringConstants.SIGNED_SIGNATURE in log_as_str)
self.assertFalse(signed_signature in log_as_str)
async def test_logging_request_and_response_body(self,
storage_account_name,
storage_account_key):
bsc = BlobServiceClient(self.account_url(storage_account_name, "blob"),
storage_account_key,
transport=AiohttpTestTransport(),
logging_enable=True)
await self._setup(bsc)
# Arrange
container = bsc.get_container_client(self.container_name)
request_body = 'testloggingbody'
blob_name = self.get_resource_name("testloggingblob")
blob_client = container.get_blob_client(blob_name)
await blob_client.upload_blob(request_body, overwrite=True)
# Act
with LogCaptured(self) as log_captured:
await blob_client.download_blob()
log_as_str = log_captured.getvalue()
self.assertFalse(request_body in log_as_str)
with LogCaptured(self) as log_captured:
await blob_client.upload_blob(request_body,
overwrite=True,
logging_body=True)
log_as_str = log_captured.getvalue()
self.assertTrue(request_body in log_as_str)
self.assertEqual(log_as_str.count(request_body), 1)
async def validate_container(client: BlobServiceClient,
container: str) -> None:
""" This makes sure that the given container exists and we have access to it.
Will raise an error which is caught further out. """
container_client = client.get_container_client(container)
props = await container_client.get_container_properties()
logging.info(props)
return None
async def test_standard_blob_tier_set_tier_api(self, resource_group,
location, storage_account,
storage_account_key):
bsc = BlobServiceClient(self.account_url(storage_account, "blob"),
credential=storage_account_key,
transport=AiohttpTestTransport())
await self._setup(bsc)
container = bsc.get_container_client(self.container_name)
tiers = [
StandardBlobTier.Archive, StandardBlobTier.Cool,
StandardBlobTier.Hot
]
for tier in tiers:
blob = self._get_blob_reference(bsc)
data = b'hello world'
await blob.upload_blob(data)
blob_ref = await blob.get_blob_properties()
self.assertIsNotNone(blob_ref.blob_tier)
self.assertTrue(blob_ref.blob_tier_inferred)
self.assertIsNone(blob_ref.blob_tier_change_time)
blobs = []
async for b in container.list_blobs():
blobs.append(b)
# Assert
self.assertIsNotNone(blobs)
self.assertGreaterEqual(len(blobs), 1)
self.assertIsNotNone(blobs[0])
self.assertNamedItemInContainer(blobs, blob.blob_name)
self.assertIsNotNone(blobs[0].blob_tier)
self.assertTrue(blobs[0].blob_tier_inferred)
self.assertIsNone(blobs[0].blob_tier_change_time)
await blob.set_standard_blob_tier(tier)
blob_ref2 = await blob.get_blob_properties()
self.assertEqual(tier, blob_ref2.blob_tier)
self.assertFalse(blob_ref2.blob_tier_inferred)
self.assertIsNotNone(blob_ref2.blob_tier_change_time)
blobs = []
async for b in container.list_blobs():
blobs.append(b)
# Assert
self.assertIsNotNone(blobs)
self.assertGreaterEqual(len(blobs), 1)
self.assertIsNotNone(blobs[0])
self.assertNamedItemInContainer(blobs, blob.blob_name)
self.assertEqual(blobs[0].blob_tier, tier)
self.assertFalse(blobs[0].blob_tier_inferred)
self.assertIsNotNone(blobs[0].blob_tier_change_time)
await blob.delete_blob()
async def test_rehydration_status(self, storage_account_name,
storage_account_key):
bsc = BlobServiceClient(self.account_url(storage_account_name, "blob"),
credential=storage_account_key,
transport=AiohttpTestTransport())
await self._setup(bsc)
blob_name = 'rehydration_test_blob_1'
blob_name2 = 'rehydration_test_blob_2'
container = bsc.get_container_client(self.container_name)
data = b'hello world'
blob = await container.upload_blob(blob_name, data)
await blob.set_standard_blob_tier(StandardBlobTier.Archive)
await blob.set_standard_blob_tier(StandardBlobTier.Cool)
blob_ref = await blob.get_blob_properties()
self.assertEqual(StandardBlobTier.Archive, blob_ref.blob_tier)
self.assertEqual("rehydrate-pending-to-cool", blob_ref.archive_status)
self.assertFalse(blob_ref.blob_tier_inferred)
blobs = []
async for b in container.list_blobs():
blobs.append(b)
await blob.delete_blob()
# Assert
self.assertIsNotNone(blobs)
self.assertGreaterEqual(len(blobs), 1)
self.assertIsNotNone(blobs[0])
self.assertNamedItemInContainer(blobs, blob.blob_name)
self.assertEqual(StandardBlobTier.Archive, blobs[0].blob_tier)
self.assertEqual("rehydrate-pending-to-cool", blobs[0].archive_status)
self.assertFalse(blobs[0].blob_tier_inferred)
blob2 = await container.upload_blob(blob_name2, data)
await blob2.set_standard_blob_tier(StandardBlobTier.Archive)
await blob2.set_standard_blob_tier(StandardBlobTier.Hot)
blob_ref2 = await blob2.get_blob_properties()
self.assertEqual(StandardBlobTier.Archive, blob_ref2.blob_tier)
self.assertEqual("rehydrate-pending-to-hot", blob_ref2.archive_status)
self.assertFalse(blob_ref2.blob_tier_inferred)
blobs = []
async for b in container.list_blobs():
blobs.append(b)
# Assert
self.assertIsNotNone(blobs)
self.assertGreaterEqual(len(blobs), 1)
self.assertIsNotNone(blobs[0])
self.assertNamedItemInContainer(blobs, blob2.blob_name)
self.assertEqual(StandardBlobTier.Archive, blobs[0].blob_tier)
self.assertEqual("rehydrate-pending-to-hot", blobs[0].archive_status)
self.assertFalse(blobs[0].blob_tier_inferred)
async def test_response_callback_async(self, resource_group, location, storage_account, storage_account_key):
# Arrange
service = BlobServiceClient(self.account_url(storage_account, "blob"), credential=storage_account_key, transport=AiohttpTestTransport())
name = self.get_resource_name('cont')
container = service.get_container_client(name)
# Act
def callback(response):
response.http_response.status_code = 200
response.http_response.headers = {}
# Assert
exists = await container.get_container_properties(raw_response_hook=callback)
self.assertTrue(exists)
async def test_authorization_is_scrubbed_off(self, storage_account_name,
storage_account_key):
bsc = BlobServiceClient(self.account_url(storage_account_name, "blob"),
storage_account_key,
transport=AiohttpTestTransport())
await self._setup(bsc)
# Arrange
container = bsc.get_container_client(self.container_name)
# Act
with LogCaptured(self) as log_captured:
await container.get_container_properties(logging_enable=True)
log_as_str = log_captured.getvalue()
# Assert
# make sure authorization header is logged, but its value is not
# the keyword SharedKey is present in the authorization header's value
self.assertTrue(_AUTHORIZATION_HEADER_NAME in log_as_str)
self.assertFalse('SharedKey' in log_as_str)
class StorageContainerTestAsync(StorageTestCase):
def setUp(self):
super(StorageContainerTestAsync, self).setUp()
url = self._get_account_url()
credential = self._get_shared_key_credential()
self.bsc = BlobServiceClient(url,
credential=credential,
transport=AiohttpTestTransport())
loop = asyncio.get_event_loop()
loop.run_until_complete(self.bsc.__aenter__())
self.test_containers = []
def tearDown(self):
if not self.is_playback():
loop = asyncio.get_event_loop()
for container_name in self.test_containers:
try:
container = self.bsc.get_container_client(container_name)
loop.run_until_complete(container.delete_container())
except HttpResponseError:
try:
lease = LeaseClient(container)
loop.run_until_complete(lease.break_lease(0))
loop.run_until_complete(container.delete_container())
except:
pass
except:
pass
loop.run_until_complete(self.bsc.__aexit__())
return super(StorageContainerTestAsync, self).tearDown()
#--Helpers-----------------------------------------------------------------
def _get_container_reference(self, prefix=TEST_CONTAINER_PREFIX):
container_name = self.get_resource_name(prefix)
self.test_containers.append(container_name)
return container_name
async def _create_container(self, prefix=TEST_CONTAINER_PREFIX):
container_name = self._get_container_reference(prefix)
container = self.bsc.get_container_client(container_name)
try:
await container.create_container()
except ResourceExistsError:
pass
return container
#--Test cases for containers -----------------------------------------
async def _test_create_container(self):
# Arrange
container_name = self._get_container_reference()
# Act
container = self.bsc.get_container_client(container_name)
created = await container.create_container()
# Assert
self.assertTrue(created)
@record
def test_create_container(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(self._test_create_container())
async def _test_create_container_with_already_existing_container_fail_on_exist(
self):
# Arrange
container_name = self._get_container_reference()
# Act
container = self.bsc.get_container_client(container_name)
created = await container.create_container()
with self.assertRaises(HttpResponseError):
await container.create_container()
# Assert
self.assertTrue(created)
@record
def test_create_container_with_already_existing_container_fail_on_exist(
self):
loop = asyncio.get_event_loop()
loop.run_until_complete(
self.
_test_create_container_with_already_existing_container_fail_on_exist(
))
async def _test_create_container_with_public_access_container(self):
# Arrange
container_name = self._get_container_reference()
# Act
container = self.bsc.get_container_client(container_name)
created = await container.create_container(public_access='container')
# Assert
self.assertTrue(created)
@record
def test_create_container_with_public_access_container(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(
self._test_create_container_with_public_access_container())
async def _test_create_container_with_public_access_blob(self):
# Arrange
container_name = self._get_container_reference()
# Act
container = self.bsc.get_container_client(container_name)
created = await container.create_container(public_access='blob')
blob = container.get_blob_client("blob1")
await blob.upload_blob(u'xyz')
anonymous_service = BlobClient(self._get_account_url(),
container=container_name,
blob="blob1")
# Assert
self.assertTrue(created)
await anonymous_service.download_blob()
@record
def test_create_container_with_public_access_blob(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(
self._test_create_container_with_public_access_blob())
async def _test_create_container_with_metadata(self):
# Arrange
container_name = self._get_container_reference()
metadata = {'hello': 'world', 'number': '42'}
# Act
container = self.bsc.get_container_client(container_name)
created = await container.create_container(metadata)
# Assert
self.assertTrue(created)
md_cr = await container.get_container_properties()
md = md_cr.metadata
self.assertDictEqual(md, metadata)
@record
def test_create_container_with_metadata(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(self._test_create_container_with_metadata())
async def _test_container_exists_with_lease(self):
# Arrange
container = await self._create_container()
await container.acquire_lease()
# Act
exists = await container.get_container_properties()
# Assert
self.assertTrue(exists)
@record
def test_container_exists_with_lease(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(self._test_container_exists_with_lease())
async def _test_unicode_create_container_unicode_name(self):
# Arrange
container_name = u'啊齄丂狛狜'
container = self.bsc.get_container_client(container_name)
# Act
with self.assertRaises(HttpResponseError):
# not supported - container name must be alphanumeric, lowercase
await container.create_container()
# Assert
@record
def test_unicode_create_container_unicode_name(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(
self._test_unicode_create_container_unicode_name())
async def _test_list_containers(self):
# Arrange
container = await self._create_container()
# Act
containers = []
async for c in self.bsc.list_containers():
containers.append(c)
# Assert
self.assertIsNotNone(containers)
self.assertGreaterEqual(len(containers), 1)
self.assertIsNotNone(containers[0])
self.assertNamedItemInContainer(containers, container.container_name)
self.assertIsNotNone(containers[0].has_immutability_policy)
self.assertIsNotNone(containers[0].has_legal_hold)
@record
def test_list_containers(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(self._test_list_containers())
async def _test_list_containers_with_prefix(self):
# Arrange
container = await self._create_container()
# Act
containers = []
async for c in self.bsc.list_containers(
name_starts_with=container.container_name):
containers.append(c)
# Assert
self.assertIsNotNone(containers)
self.assertEqual(len(containers), 1)
self.assertIsNotNone(containers[0])
self.assertEqual(containers[0].name, container.container_name)
self.assertIsNone(containers[0].metadata)
@record
def test_list_containers_with_prefix(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(self._test_list_containers_with_prefix())
async def _test_list_containers_with_include_metadata(self):
# Arrange
container = await self._create_container()
metadata = {'hello': 'world', 'number': '42'}
resp = await container.set_container_metadata(metadata)
# Act
containers = []
async for c in self.bsc.list_containers(
name_starts_with=container.container_name,
include_metadata=True):
containers.append(c)
# Assert
self.assertIsNotNone(containers)
self.assertGreaterEqual(len(containers), 1)
self.assertIsNotNone(containers[0])
self.assertNamedItemInContainer(containers, container.container_name)
self.assertDictEqual(containers[0].metadata, metadata)
@record
def test_list_containers_with_include_metadata(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(
self._test_list_containers_with_include_metadata())
async def _test_list_containers_with_public_access(self):
# Arrange
container = await self._create_container()
resp = await container.set_container_access_policy(
public_access=PublicAccess.Blob)
# Act
containers = []
async for c in self.bsc.list_containers(
name_starts_with=container.container_name):
containers.append(c)
# Assert
self.assertIsNotNone(containers)
self.assertGreaterEqual(len(containers), 1)
self.assertIsNotNone(containers[0])
self.assertNamedItemInContainer(containers, container.container_name)
self.assertEqual(containers[0].public_access, PublicAccess.Blob)
@record
def test_list_containers_with_public_access(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(
self._test_list_containers_with_public_access())
async def _test_list_containers_with_num_results_and_marker(self):
# Arrange
prefix = 'listcontainer'
container_names = []
for i in range(0, 4):
cr = await self._create_container(prefix + str(i))
container_names.append(cr.container_name)
container_names.sort()
# Act
generator1 = self.bsc.list_containers(name_starts_with=prefix,
results_per_page=2).by_page()
containers1 = []
async for c in await generator1.__anext__():
containers1.append(c)
generator2 = self.bsc.list_containers(
name_starts_with=prefix,
results_per_page=2).by_page(generator1.continuation_token)
containers2 = []
async for c in await generator2.__anext__():
containers2.append(c)
# Assert
self.assertIsNotNone(containers1)
self.assertEqual(len(containers1), 2)
self.assertNamedItemInContainer(containers1, container_names[0])
self.assertNamedItemInContainer(containers1, container_names[1])
self.assertIsNotNone(containers2)
self.assertEqual(len(containers2), 2)
self.assertNamedItemInContainer(containers2, container_names[2])
self.assertNamedItemInContainer(containers2, container_names[3])
@record
def test_list_containers_with_num_results_and_marker(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(
self._test_list_containers_with_num_results_and_marker())
async def _test_set_container_metadata(self):
# Arrange
metadata = {'hello': 'world', 'number': '43'}
container = await self._create_container()
# Act
await container.set_container_metadata(metadata)
md = await container.get_container_properties()
metadata_from_response = md.metadata
# Assert
self.assertDictEqual(metadata_from_response, metadata)
@record
def test_set_container_metadata(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(self._test_set_container_metadata())
async def _test_set_container_metadata_with_lease_id(self):
# Arrange
metadata = {'hello': 'world', 'number': '43'}
container = await self._create_container()
lease_id = await container.acquire_lease()
# Act
await container.set_container_metadata(metadata, lease_id)
# Assert
md = await container.get_container_properties()
md = md.metadata
self.assertDictEqual(md, metadata)
@record
def test_set_container_metadata_with_lease_id(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(
self._test_set_container_metadata_with_lease_id())
async def _test_set_container_metadata_with_non_existing_container(self):
# Arrange
container_name = self._get_container_reference()
container = self.bsc.get_container_client(container_name)
# Act
with self.assertRaises(ResourceNotFoundError):
await container.set_container_metadata({
'hello': 'world',
'number': '43'
})
# Assert
@record
def test_set_container_metadata_with_non_existing_container(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(
self._test_set_container_metadata_with_non_existing_container())
async def _test_get_container_metadata(self):
# Arrange
metadata = {'hello': 'world', 'number': '42'}
container = await self._create_container()
await container.set_container_metadata(metadata)
# Act
md_cr = await container.get_container_properties()
md = md_cr.metadata
# Assert
self.assertDictEqual(md, metadata)
@record
def test_get_container_metadata(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(self._test_get_container_metadata())
async def _test_get_container_metadata_with_lease_id(self):
# Arrange
metadata = {'hello': 'world', 'number': '42'}
container = await self._create_container()
await container.set_container_metadata(metadata)
lease_id = await container.acquire_lease()
# Act
md = await container.get_container_properties(lease_id)
md = md.metadata
# Assert
self.assertDictEqual(md, metadata)
@record
def test_get_container_metadata_with_lease_id(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(
self._test_get_container_metadata_with_lease_id())
async def _test_get_container_properties(self):
# Arrange
metadata = {'hello': 'world', 'number': '42'}
container = await self._create_container()
await container.set_container_metadata(metadata)
# Act
props = await container.get_container_properties()
# Assert
self.assertIsNotNone(props)
self.assertDictEqual(props.metadata, metadata)
# self.assertEqual(props.lease.duration, 'infinite')
# self.assertEqual(props.lease.state, 'leased')
# self.assertEqual(props.lease.status, 'locked')
# self.assertEqual(props.public_access, 'container')
self.assertIsNotNone(props.has_immutability_policy)
self.assertIsNotNone(props.has_legal_hold)
@record
def test_get_container_properties(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(self._test_get_container_properties())
async def _test_get_container_properties_with_lease_id(self):
# Arrange
metadata = {'hello': 'world', 'number': '42'}
container = await self._create_container()
await container.set_container_metadata(metadata)
lease_id = await container.acquire_lease()
# Act
props = await container.get_container_properties(lease_id)
await lease_id.break_lease()
# Assert
self.assertIsNotNone(props)
self.assertDictEqual(props.metadata, metadata)
self.assertEqual(props.lease.duration, 'infinite')
self.assertEqual(props.lease.state, 'leased')
self.assertEqual(props.lease.status, 'locked')
@record
def test_get_container_properties_with_lease_id(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(
self._test_get_container_properties_with_lease_id())
async def _test_get_container_acl(self):
# Arrange
container = await self._create_container()
# Act
acl = await container.get_container_access_policy()
# Assert
self.assertIsNotNone(acl)
self.assertIsNone(acl.get('public_access'))
self.assertEqual(len(acl.get('signed_identifiers')), 0)
@record
def test_get_container_acl(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(self._test_get_container_acl())
async def _test_get_container_acl_with_lease_id(self):
# Arrange
container = await self._create_container()
lease_id = await container.acquire_lease()
# Act
acl = await container.get_container_access_policy(lease_id)
# Assert
self.assertIsNotNone(acl)
self.assertIsNone(acl.get('public_access'))
@record
def test_get_container_acl_with_lease_id(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(self._test_get_container_acl_with_lease_id())
async def _test_set_container_acl(self):
# Arrange
container = await self._create_container()
# Act
response = await container.set_container_access_policy()
self.assertIsNotNone(response.get('etag'))
self.assertIsNotNone(response.get('last_modified'))
# Assert
acl = await container.get_container_access_policy()
self.assertIsNotNone(acl)
self.assertEqual(len(acl.get('signed_identifiers')), 0)
self.assertIsNone(acl.get('public_access'))
@record
def test_set_container_acl(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(self._test_set_container_acl())
async def _test_set_container_acl_with_one_signed_identifier(self):
# Arrange
from dateutil.tz import tzutc
container = await self._create_container()
# Act
access_policy = AccessPolicy(permission=ContainerPermissions.READ,
expiry=datetime.utcnow() +
timedelta(hours=1),
start=datetime.utcnow())
signed_identifier = {'testid': access_policy}
response = await container.set_container_access_policy(
signed_identifier)
# Assert
self.assertIsNotNone(response.get('etag'))
self.assertIsNotNone(response.get('last_modified'))
@record
def test_set_container_acl_with_one_signed_identifier(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(
self._test_set_container_acl_with_one_signed_identifier())
async def _test_set_container_acl_with_lease_id(self):
# Arrange
container = await self._create_container()
lease_id = await container.acquire_lease()
# Act
await container.set_container_access_policy(lease=lease_id)
# Assert
acl = await container.get_container_access_policy()
self.assertIsNotNone(acl)
self.assertIsNone(acl.get('public_access'))
@record
def test_set_container_acl_with_lease_id(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(self._test_set_container_acl_with_lease_id())
async def _test_set_container_acl_with_public_access(self):
# Arrange
container = await self._create_container()
# Act
await container.set_container_access_policy(public_access='container')
# Assert
acl = await container.get_container_access_policy()
self.assertIsNotNone(acl)
self.assertEqual('container', acl.get('public_access'))
@record
def test_set_container_acl_with_public_access(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(
self._test_set_container_acl_with_public_access())
async def _test_set_container_acl_with_empty_signed_identifiers(self):
# Arrange
container = await self._create_container()
# Act
await container.set_container_access_policy(signed_identifiers=dict())
# Assert
acl = await container.get_container_access_policy()
self.assertIsNotNone(acl)
self.assertEqual(len(acl.get('signed_identifiers')), 0)
self.assertIsNone(acl.get('public_access'))
@record
def test_set_container_acl_with_empty_signed_identifiers(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(
self._test_set_container_acl_with_empty_signed_identifiers())
async def _test_set_container_acl_with_signed_identifiers(self):
# Arrange
container = await self._create_container()
# Act
access_policy = AccessPolicy(
permission=ContainerPermissions.READ,
expiry=datetime.utcnow() + timedelta(hours=1),
start=datetime.utcnow() - timedelta(minutes=1))
identifiers = {'testid': access_policy}
await container.set_container_access_policy(identifiers)
# Assert
acl = await container.get_container_access_policy()
self.assertIsNotNone(acl)
self.assertEqual('testid', acl.get('signed_identifiers')[0].id)
self.assertIsNone(acl.get('public_access'))
@record
def test_set_container_acl_with_signed_identifiers(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(
self._test_set_container_acl_with_signed_identifiers())
async def _test_set_container_acl_with_empty_identifiers(self):
# Arrange
container = await self._create_container()
identifiers = {i: None for i in range(0, 3)}
# Act
await container.set_container_access_policy(identifiers)
# Assert
acl = await container.get_container_access_policy()
self.assertIsNotNone(acl)
self.assertEqual(len(acl.get('signed_identifiers')), 3)
self.assertEqual('0', acl.get('signed_identifiers')[0].id)
self.assertIsNone(acl.get('signed_identifiers')[0].access_policy)
self.assertIsNone(acl.get('public_access'))
@record
def test_set_container_acl_with_empty_identifiers(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(
self._test_set_container_acl_with_empty_identifiers())
async def _test_set_container_acl_with_three_identifiers(self):
# Arrange
container = await self._create_container()
access_policy = AccessPolicy(
permission=ContainerPermissions.READ,
expiry=datetime.utcnow() + timedelta(hours=1),
start=datetime.utcnow() - timedelta(minutes=1))
identifiers = {i: access_policy for i in range(2)}
# Act
await container.set_container_access_policy(identifiers)
# Assert
acl = await container.get_container_access_policy()
self.assertIsNotNone(acl)
self.assertEqual(len(acl.get('signed_identifiers')), 2)
self.assertEqual('0', acl.get('signed_identifiers')[0].id)
self.assertIsNotNone(acl.get('signed_identifiers')[0].access_policy)
self.assertIsNone(acl.get('public_access'))
@record
def test_set_container_acl_with_three_identifiers(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(
self._test_set_container_acl_with_three_identifiers())
async def _test_set_container_acl_too_many_ids(self):
# Arrange
container_name = await self._create_container()
# Act
identifiers = dict()
for i in range(0, 6):
identifiers['id{}'.format(i)] = AccessPolicy()
# Assert
with self.assertRaises(ValueError) as e:
await container_name.set_container_access_policy(identifiers)
self.assertEqual(
str(e.exception),
'Too many access policies provided. The server does not support setting more than 5 access policies on a single resource.'
)
@record
def test_set_container_acl_too_many_ids(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(self._test_set_container_acl_too_many_ids())
async def _test_lease_container_acquire_and_release(self):
# Arrange
container = await self._create_container()
# Act
lease = await container.acquire_lease()
await lease.release()
# Assert
@record
def test_lease_container_acquire_and_release(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(
self._test_lease_container_acquire_and_release())
async def _test_lease_container_renew(self):
# Arrange
container = await self._create_container()
lease = await container.acquire_lease(lease_duration=15)
self.sleep(10)
lease_id_start = lease.id
# Act
await lease.renew()
# Assert
self.assertEqual(lease.id, lease_id_start)
self.sleep(5)
with self.assertRaises(HttpResponseError):
await container.delete_container()
self.sleep(10)
await container.delete_container()
@record
def test_lease_container_renew(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(self._test_lease_container_renew())
async def _test_lease_container_break_period(self):
# Arrange
container = await self._create_container()
# Act
lease = await container.acquire_lease(lease_duration=15)
# Assert
await lease.break_lease(lease_break_period=5)
self.sleep(6)
with self.assertRaises(HttpResponseError):
await container.delete_container(lease=lease)
@record
def test_lease_container_break_period(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(self._test_lease_container_break_period())
async def _test_lease_container_break_released_lease_fails(self):
# Arrange
container = await self._create_container()
lease = await container.acquire_lease()
await lease.release()
# Act
with self.assertRaises(HttpResponseError):
await lease.break_lease()
# Assert
@record
def test_lease_container_break_released_lease_fails(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(
self._test_lease_container_break_released_lease_fails())
async def _test_lease_container_with_duration(self):
# Arrange
container = await self._create_container()
# Act
lease = await container.acquire_lease(lease_duration=15)
# Assert
with self.assertRaises(HttpResponseError):
await container.acquire_lease()
self.sleep(15)
await container.acquire_lease()
@record
def test_lease_container_with_duration(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(self._test_lease_container_with_duration())
async def _test_lease_container_twice(self):
# Arrange
container = await self._create_container()
# Act
lease = await container.acquire_lease(lease_duration=15)
# Assert
lease2 = await container.acquire_lease(lease_id=lease.id)
self.assertEqual(lease.id, lease2.id)
@record
def test_lease_container_twice(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(self._test_lease_container_twice())
async def _test_lease_container_with_proposed_lease_id(self):
# Arrange
container = await self._create_container()
# Act
proposed_lease_id = '55e97f64-73e8-4390-838d-d9e84a374321'
lease = await container.acquire_lease(lease_id=proposed_lease_id)
# Assert
self.assertEqual(proposed_lease_id, lease.id)
@record
def test_lease_container_with_proposed_lease_id(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(
self._test_lease_container_with_proposed_lease_id())
async def _test_lease_container_change_lease_id(self):
# Arrange
container = await self._create_container()
# Act
lease_id = '29e0b239-ecda-4f69-bfa3-95f6af91464c'
lease = await container.acquire_lease()
lease_id1 = lease.id
await lease.change(proposed_lease_id=lease_id)
await lease.renew()
lease_id2 = lease.id
# Assert
self.assertIsNotNone(lease_id1)
self.assertIsNotNone(lease_id2)
self.assertNotEqual(lease_id1, lease_id)
self.assertEqual(lease_id2, lease_id)
@record
def test_lease_container_change_lease_id(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(self._test_lease_container_change_lease_id())
async def _test_delete_container_with_existing_container(self):
# Arrange
container = await self._create_container()
# Act
deleted = await container.delete_container()
# Assert
self.assertIsNone(deleted)
@record
def test_delete_container_with_existing_container(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(
self._test_delete_container_with_existing_container())
async def _test_delete_container_with_non_existing_container_fail_not_exist(
self):
# Arrange
container_name = self._get_container_reference()
container = self.bsc.get_container_client(container_name)
# Act
with LogCaptured(self) as log_captured:
with self.assertRaises(ResourceNotFoundError):
await container.delete_container()
log_as_str = log_captured.getvalue()
#self.assertTrue('ERROR' in log_as_str)
@record
def test_delete_container_with_non_existing_container_fail_not_exist(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(
self.
_test_delete_container_with_non_existing_container_fail_not_exist(
))
async def _test_delete_container_with_lease_id(self):
# Arrange
container = await self._create_container()
lease = await container.acquire_lease(lease_duration=15)
# Act
deleted = await container.delete_container(lease=lease)
# Assert
self.assertIsNone(deleted)
with self.assertRaises(ResourceNotFoundError):
await container.get_container_properties()
@record
def test_delete_container_with_lease_id(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(self._test_delete_container_with_lease_id())
async def _test_list_names(self):
# Arrange
container = await self._create_container()
data = b'hello world'
await (container.get_blob_client('blob1')).upload_blob(data)
await (container.get_blob_client('blob2')).upload_blob(data)
# Act
blobs = []
async for b in container.list_blobs():
blobs.append(b.name)
self.assertEqual(blobs, ['blob1', 'blob2'])
@record
def test_list_names(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(self._test_list_names())
async def _test_list_blobs(self):
# Arrange
container = await self._create_container()
data = b'hello world'
cr0 = container.get_blob_client('blob1')
await cr0.upload_blob(data)
cr1 = container.get_blob_client('blob2')
await cr1.upload_blob(data)
# Act
blobs = []
async for b in container.list_blobs():
blobs.append(b)
# Assert
self.assertIsNotNone(blobs)
self.assertGreaterEqual(len(blobs), 2)
self.assertIsNotNone(blobs[0])
self.assertNamedItemInContainer(blobs, 'blob1')
self.assertNamedItemInContainer(blobs, 'blob2')
self.assertEqual(blobs[0].size, 11)
self.assertEqual(blobs[1].content_settings.content_type,
'application/octet-stream')
self.assertIsNotNone(blobs[0].creation_time)
@record
def test_list_blobs(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(self._test_list_blobs())
async def _test_list_blobs_leased_blob(self):
# Arrange
container = await self._create_container()
data = b'hello world'
blob1 = container.get_blob_client('blob1')
await blob1.upload_blob(data)
lease = await blob1.acquire_lease()
# Act
resp = []
async for b in container.list_blobs():
resp.append(b)
# Assert
self.assertIsNotNone(resp)
self.assertGreaterEqual(len(resp), 1)
self.assertIsNotNone(resp[0])
self.assertNamedItemInContainer(resp, 'blob1')
self.assertEqual(resp[0].size, 11)
self.assertEqual(resp[0].lease.duration, 'infinite')
self.assertEqual(resp[0].lease.status, 'locked')
self.assertEqual(resp[0].lease.state, 'leased')
@record
def test_list_blobs_leased_blob(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(self._test_list_blobs_leased_blob())
async def _test_list_blobs_with_prefix(self):
# Arrange
container = await self._create_container()
data = b'hello world'
c0 = container.get_blob_client('blob_a1')
await c0.upload_blob(data)
c1 = container.get_blob_client('blob_a2')
await c1.upload_blob(data)
c2 = container.get_blob_client('blob_b1')
await c2.upload_blob(data)
# Act
resp = []
async for b in container.list_blobs(name_starts_with='blob_a'):
resp.append(b)
# Assert
self.assertIsNotNone(resp)
self.assertEqual(len(resp), 2)
self.assertNamedItemInContainer(resp, 'blob_a1')
self.assertNamedItemInContainer(resp, 'blob_a2')
@record
def test_list_blobs_with_prefix(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(self._test_list_blobs_with_prefix())
async def _test_list_blobs_with_num_results(self):
# Arrange
container = await self._create_container()
data = b'hello world'
c0 = container.get_blob_client('blob_a1')
await c0.upload_blob(data)
c1 = container.get_blob_client('blob_a2')
await c1.upload_blob(data)
c2 = container.get_blob_client('blob_a3')
await c2.upload_blob(data)
c3 = container.get_blob_client('blob_b1')
await c3.upload_blob(data)
# Act
generator = container.list_blobs(results_per_page=2).by_page()
blobs = []
async for b in await generator.__anext__():
blobs.append(b)
# Assert
self.assertIsNotNone(blobs)
self.assertEqual(len(blobs), 2)
self.assertNamedItemInContainer(generator.current_page, 'blob_a1')
self.assertNamedItemInContainer(generator.current_page, 'blob_a2')
@record
def test_list_blobs_with_num_results(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(self._test_list_blobs_with_num_results())
async def _test_list_blobs_with_include_snapshots(self):
# Arrange
container = await self._create_container()
data = b'hello world'
blob1 = container.get_blob_client('blob1')
await blob1.upload_blob(data)
await blob1.create_snapshot()
await (container.get_blob_client('blob2')).upload_blob(data)
# Act
blobs = []
async for b in container.list_blobs(include="snapshots"):
blobs.append(b)
# Assert
self.assertEqual(len(blobs), 3)
self.assertEqual(blobs[0].name, 'blob1')
self.assertIsNotNone(blobs[0].snapshot)
self.assertEqual(blobs[1].name, 'blob1')
self.assertIsNone(blobs[1].snapshot)
self.assertEqual(blobs[2].name, 'blob2')
self.assertIsNone(blobs[2].snapshot)
@record
def test_list_blobs_with_include_snapshots(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(self._test_list_blobs_with_include_snapshots())
async def _test_list_blobs_with_include_metadata(self):
# Arrange
pytest.skip("Waiting on metadata XML fix in msrest")
container = await self._create_container()
data = b'hello world'
blob1 = container.get_blob_client('blob1')
await blob1.upload_blob(data, metadata={'number': '1', 'name': 'bob'})
await blob1.create_snapshot()
cr = container.get_blob_client('blob2')
await cr.upload_blob(data, metadata={'number': '2', 'name': 'car'})
# Act
blobs = []
async for b in container.list_blobs(include="metadata"):
blobs.append(b)
# Assert
self.assertEqual(len(blobs), 2)
self.assertEqual(blobs[0].name, 'blob1')
self.assertEqual(blobs[0].metadata['number'], '1')
self.assertEqual(blobs[0].metadata['name'], 'bob')
self.assertEqual(blobs[1].name, 'blob2')
self.assertEqual(blobs[1].metadata['number'], '2')
self.assertEqual(blobs[1].metadata['name'], 'car')
@record
def test_list_blobs_with_include_metadata(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(self._test_list_blobs_with_include_metadata())
async def _test_list_blobs_with_include_uncommittedblobs(self):
# Arrange
container = await self._create_container()
data = b'hello world'
blob1 = container.get_blob_client('blob1')
await blob1.stage_block('1', b'AAA')
await blob1.stage_block('2', b'BBB')
await blob1.stage_block('3', b'CCC')
blob2 = container.get_blob_client('blob2')
await blob2.upload_blob(data, metadata={'number': '2', 'name': 'car'})
# Act
blobs = []
async for b in container.list_blobs(include="uncommittedblobs"):
blobs.append(b)
# Assert
self.assertEqual(len(blobs), 2)
self.assertEqual(blobs[0].name, 'blob1')
self.assertEqual(blobs[1].name, 'blob2')
@record
def test_list_blobs_with_include_uncommittedblobs(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(
self._test_list_blobs_with_include_uncommittedblobs())
async def _test_list_blobs_with_include_copy(self):
# Arrange
container = await self._create_container()
data = b'hello world'
await (container.get_blob_client('blob1')).upload_blob(
data, metadata={'status': 'original'})
sourceblob = 'https://{0}.blob.core.windows.net/{1}/blob1'.format(
self.settings.STORAGE_ACCOUNT_NAME, container.container_name)
blobcopy = container.get_blob_client('blob1copy')
await blobcopy.start_copy_from_url(sourceblob,
metadata={'status': 'copy'})
# Act
blobs = []
async for b in container.list_blobs(include="copy"):
blobs.append(b)
# Assert
self.assertEqual(len(blobs), 2)
self.assertEqual(blobs[0].name, 'blob1')
self.assertEqual(blobs[1].name, 'blob1copy')
self.assertEqual(blobs[1].blob_type, blobs[0].blob_type)
self.assertEqual(blobs[1].size, 11)
self.assertEqual(blobs[1].content_settings.content_type,
'application/octet-stream')
self.assertEqual(blobs[1].content_settings.cache_control, None)
self.assertEqual(blobs[1].content_settings.content_encoding, None)
self.assertEqual(blobs[1].content_settings.content_language, None)
self.assertEqual(blobs[1].content_settings.content_disposition, None)
self.assertNotEqual(blobs[1].content_settings.content_md5, None)
self.assertEqual(blobs[1].lease.status, 'unlocked')
self.assertEqual(blobs[1].lease.state, 'available')
self.assertNotEqual(blobs[1].copy.id, None)
self.assertEqual(blobs[1].copy.source, sourceblob)
self.assertEqual(blobs[1].copy.status, 'success')
self.assertEqual(blobs[1].copy.progress, '11/11')
self.assertNotEqual(blobs[1].copy.completion_time, None)
@record
def test_list_blobs_with_include_copy(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(self._test_list_blobs_with_include_copy())
async def _test_list_blobs_with_delimiter(self):
# Arrange
container = await self._create_container()
data = b'hello world'
cr0 = container.get_blob_client('a/blob1')
await cr0.upload_blob(data)
cr1 = container.get_blob_client('a/blob2')
await cr1.upload_blob(data)
cr2 = container.get_blob_client('b/blob3')
await cr2.upload_blob(data)
cr4 = container.get_blob_client('blob4')
await cr4.upload_blob(data)
# Act
resp = []
async for w in container.walk_blobs():
resp.append(w)
# Assert
self.assertIsNotNone(resp)
self.assertEqual(len(resp), 3)
self.assertNamedItemInContainer(resp, 'a/')
self.assertNamedItemInContainer(resp, 'b/')
self.assertNamedItemInContainer(resp, 'blob4')
@record
def test_list_blobs_with_delimiter(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(self._test_list_blobs_with_delimiter())
async def _test_walk_blobs_with_delimiter(self):
# Arrange
container = await self._create_container()
data = b'hello world'
cr0 = container.get_blob_client('a/blob1')
await cr0.upload_blob(data)
cr1 = container.get_blob_client('a/blob2')
await cr1.upload_blob(data)
cr2 = container.get_blob_client('b/c/blob3')
await cr2.upload_blob(data)
cr3 = container.get_blob_client('blob4')
await cr3.upload_blob(data)
blob_list = []
def recursive_walk(prefix):
for b in prefix:
if b.get('prefix'):
recursive_walk(b)
else:
blob_list.append(b.name)
# Act
recursive_walk(container.walk_blobs())
# Assert
self.assertEqual(len(blob_list), 4)
self.assertEqual(blob_list,
['a/blob1', 'a/blob2', 'b/c/blob3', 'blob4'])
@pytest.mark.skip
def test_walk_blobs_with_delimiter(self):
if TestMode.need_recording_file(self.test_mode):
return
loop = asyncio.get_event_loop()
loop.run_until_complete(self._test_walk_blobs_with_delimiter())
async def _test_list_blobs_with_include_multiple(self):
# Arrange
pytest.skip("Waiting on metadata XML fix in msrest")
container = await self._create_container()
data = b'hello world'
blob1 = container.get_blob_client('blob1')
await blob1.upload_blob(data, metadata={'number': '1', 'name': 'bob'})
await blob1.create_snapshot()
client = container.get_blob_client('blob2')
await client.upload_blob(data, metadata={'number': '2', 'name': 'car'})
# Act
blobs = []
async for b in container.list_blobs(include=["snapshots", "metadata"]):
blobs.append(b)
# Assert
self.assertEqual(len(blobs), 3)
self.assertEqual(blobs[0].name, 'blob1')
self.assertIsNotNone(blobs[0].snapshot)
self.assertEqual(blobs[0].metadata['number'], '1')
self.assertEqual(blobs[0].metadata['name'], 'bob')
self.assertEqual(blobs[1].name, 'blob1')
self.assertIsNone(blobs[1].snapshot)
self.assertEqual(blobs[1].metadata['number'], '1')
self.assertEqual(blobs[1].metadata['name'], 'bob')
self.assertEqual(blobs[2].name, 'blob2')
self.assertIsNone(blobs[2].snapshot)
self.assertEqual(blobs[2].metadata['number'], '2')
self.assertEqual(blobs[2].metadata['name'], 'car')
@record
def test_list_blobs_with_include_multiple(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(self._test_list_blobs_with_include_multiple())
async def _test_shared_access_container(self):
# SAS URL is calculated from storage key, so this test runs live only
if TestMode.need_recording_file(self.test_mode):
return
# Arrange
container = await self._create_container()
blob_name = 'blob1'
data = b'hello world'
blob = container.get_blob_client(blob_name)
await blob.upload_blob(data)
token = container.generate_shared_access_signature(
expiry=datetime.utcnow() + timedelta(hours=1),
permission=ContainerPermissions.READ,
)
blob = BlobClient(blob.url, credential=token)
# Act
response = requests.get(blob.url)
# Assert
self.assertTrue(response.ok)
self.assertEqual(data, response.content)
@record
def test_shared_access_container(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(self._test_shared_access_container())
async def _test_web_container_normal_operations_working(self):
web_container = "web"
# create the web container in case it does not exist yet
container = self.bsc.get_container_client(web_container)
try:
try:
created = await container.create_container()
self.assertIsNotNone(created)
except ResourceExistsError:
pass
# test if web container exists
exist = await container.get_container_properties()
self.assertTrue(exist)
# create a blob
blob_name = self.get_resource_name("blob")
blob_content = self.get_random_text_data(1024)
blob = container.get_blob_client(blob_name)
await blob.upload_blob(blob_content)
# get a blob
blob_data = await (await blob.download_blob()).content_as_bytes()
self.assertIsNotNone(blob)
self.assertEqual(blob_data.decode('utf-8'), blob_content)
finally:
# delete container
await container.delete_container()
@record
def test_web_container_normal_operations_working(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(
self._test_web_container_normal_operations_working())
class AzBlobManagerAsync:
"""A utility class to help working with Azure Blob Storage.
This class implements asynchronous methods based on the
Microsoft Python SDK azure.storage.blob.aio
See:
https://docs.microsoft.com/en-us/python/api/azure-storage-blob/azure.storage.blob.aio?view=azure-python
Available:
- Basic methods to work with containers and blobs
"""
@classmethod
def create(cls, connection_string=None, account_url=None, credential=None):
"""Instantiate an asynchronous AzBlobManagerAsync object.
Args:
connection_string (str): A connection string to an Azure Storage account.
account_url (str): The URL to the blob storage account. Any other entities included in
the URL path (e.g. container or blob) will be discarded. This URL can be
optionally authenticated with a SAS token.
credential (str): The credentials with which to authenticate. This is optional
if the account URL already has a SAS token, or the connection string already
has shared access key values. The value can be a SAS token string, an account
shared access key, or an instance of a TokenCredentials class from azure.identity.
Credentials provided here will take precedence over those in the connection string.
Returns:
AzBlobManagerAsync object
Examples:
Creating the AzBlobManagerAsync with account url and a shared access key:
azStorageManager = AzBlobManagerAsync.create(account_url=self.url, credential=self.shared_access_key)
Creating the AzBlobManagerAsync with a connection string that has the shared access key:
azStorageManager = AzBlobManagerAsync.create(connection_string='DefaultEndpointsProtocol=http;...')
"""
self = AzBlobManagerAsync()
self.connection_string = connection_string
self.account_url = account_url
self.credential = credential
from azure.storage.blob.aio import BlobServiceClient
self.blob_service_client = BlobServiceClient
if (self.connection_string is not None):
# Create BlobServiceClient from a Connection String
self.blob_service_client = BlobServiceClient.from_connection_string(
conn_str=self.connection_string, credential=self.credential)
else:
# Creating the BlobServiceClient with account url and credential.
self.blob_service_client = BlobServiceClient(
account_url=self.account_url, credential=self.credential)
return self
def _logAzureError(self, err=AzureError):
msg = err.message.split('\n')[0]
logger.error(f'AzureError error: {msg}')
async def create_container(self, container_name):
"""Creates a new container.
Args:
container_name (str): The name of the container.
See https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata
for naming convention
Returns:
bool: The return value. True for success, False otherwise.
"""
success = False
try:
new_container = await self.blob_service_client.create_container(
container_name)
properties = await new_container.get_container_properties()
success = properties is not None and properties.name == container_name
except ResourceExistsError:
logger.info(f'Container \"{container_name}\" already exists.')
except AzureError as err:
self._logAzureError(err=err)
except Exception:
logger.exception('')
return success
async def delete_container(self, container_name):
"""Deletes a container.
Args:
container_name (str): The name of the container.
Returns:
bool: The return value. True for success, False otherwise.
"""
success = False
try:
await self.blob_service_client.delete_container(container_name)
success = True
except ResourceNotFoundError:
logger.info(f'Container \"{container_name}\" doesn not exist.')
except AzureError as err:
self._logAzureError(err=err)
except Exception:
logger.exception('')
return success
async def _list_containers(self,
name_starts_with=None,
include_metadata=False):
"""Lists containers.
Args:
name_starts_with (str): Filters the results to return only containers whose names
begin with the specified prefix.
include_metadata (bool): Specifies that container metadata to be returned in the response.
Returns:
ItemPaged[ContainerProperties]: An iterable (auto-paging) of ContainerProperties.
"""
try:
containers = []
async for container in self.blob_service_client.list_containers(
name_starts_with=name_starts_with,
include_metadata=include_metadata):
containers.append(container)
return containers
except AzureError as err:
self._logAzureError(err=err)
except Exception:
logger.exception('')
return None
async def list_containers_name(self, name_starts_with=None):
"""Lists containers' name.
Args:
name_starts_with (str): Filters the results to return only containers whose names
begin with the specified prefix.
Returns:
list: A list of strings representing the container names.
"""
containers_list = []
containers = await self._list_containers(
name_starts_with=name_starts_with, include_metadata=False)
if (containers is None):
return containers_list
for container in containers:
containers_list.append(container['name'])
return containers_list
async def create_append_blob(self,
container_name,
blob_name,
replace_blob=False):
"""Creates an append blob in an existing container.
Args:
container_name (str): The name of the container.
blob_name (str): The name of the blob.
replace_blob (bool): If True, deletes existing blob with same name
Returns:
bool: The return value. True for success, False otherwise.
"""
success = False
try:
blob_client = self.blob_service_client.get_blob_client(
container_name, blob_name)
# raise ResourceNotFoundError if blob does not exist
await blob_client.get_blob_properties()
# blob exists already
if (replace_blob is True):
await blob_client.create_append_blob()
success = True
except ResourceNotFoundError:
await blob_client.create_append_blob()
success = True
except AzureError as err:
self._logAzureError(err=err)
except Exception:
logger.exception('')
return success
async def create_page_blob(self,
container_name,
blob_name,
size=1024,
content_settings=None,
metadata=None,
premium_page_blob_tier=None):
"""Creates a page blob in an existing container.
Args:
container_name (str): The name of the container.
blob_name (str): The name of the blob.
size (int): This specifies the maximum size for the page blob, up to 1 TB.
The page blob size must be aligned to a 512-byte boundary
content_settings (ContentSettings): ContentSettings object used to set blob properties.
Used to set content type, encoding, language, disposition, md5, and cache control.
metadata (dict(str, str)): Name-value pairs associated with the blob as metadata
premium_page_blob_tier (PremiumPageBlobTier): A page blob tier value to set the blob to
Returns:
bool: The return value. True for success, False otherwise.
"""
success = False
try:
blob_client = self.blob_service_client.get_blob_client(
container_name, blob_name)
await blob_client.create_page_blob(size, content_settings,
metadata,
premium_page_blob_tier)
success = True
except AzureError as err:
self._logAzureError(err=err)
except Exception:
logger.exception('')
return success
async def delete_blob(self, container_name, blob_name):
"""Deletes a blob.
Args:
container_name (str): The name of the container.
blob_name (str): The name of the blob.
Returns:
bool: The return value. True for success, False otherwise.
"""
success = False
try:
blob_client = self.blob_service_client.get_blob_client(
container_name, blob_name)
await blob_client.delete_blob()
success = True
except AzureError as err:
self._logAzureError(err=err)
except Exception:
logger.exception('')
return success
async def list_blobs(self, container_name):
"""Lists the blobs in the specified container.
Args:
container_name (str): The name of the container.
Returns:
list: A list of strings representing the blob names.
"""
blobs_list = []
try:
container_client = self.blob_service_client.get_container_client(
container_name)
async for blob in container_client.list_blobs():
blobs_list.append(blob)
except AzureError as err:
self._logAzureError(err=err)
except Exception:
logger.exception(f'Fatal error')
return blobs_list
async def upload_data(self,
data,
container_name,
blob_name,
blob_type='BlockBlob'):
"""Creates a new blob from a data source with automatic chunking.
Args:
data: The blob data to upload.
container_name (str): The name of the container.
blob_name (str): The name of the blob.
blob_typr (str): The type of the blob. This can be either BlockBlob, PageBlob or AppendBlob.
Returns:
bool: The return value. True for success, False otherwise.
"""
success = False
try:
blob_client = self.blob_service_client.get_blob_client(
container_name, blob_name)
await blob_client.upload_blob(data)
success = True
except AzureError as err:
self._logAzureError(err=err)
except Exception:
logger.exception('')
return success
async def append_block(self, data, container_name, blob_name):
"""Commits a new block of data to the end of the existing append blob.
Args:
data: Content of the block.
container_name (str): The name of the container.
blob_name (str): The name of the blob.
Returns:
bool: The return value. True for success, False otherwise.
"""
success = False
try:
blob_client = self.blob_service_client.get_blob_client(
container_name, blob_name)
await blob_client.append_block(data)
success = True
except AzureError as err:
self._logAzureError(err=err)
except Exception:
logger.exception('')
return success
async def download_data(self, container_name, blob_name):
"""Downloads a blob.
Args:
container_name (str): The name of the container.
blob_name (str): The name of the blob.
Returns:
stream: The data stream
"""
try:
blob_client = self.blob_service_client.get_blob_client(
container_name, blob_name)
stream = await blob_client.download_blob()
return await stream.readall()
except AzureError as err:
self._logAzureError(err=err)
except Exception:
logger.exception('')
class StorageBlobEncryptionTestAsync(AsyncStorageTestCase):
# --Helpers-----------------------------------------------------------------
async def _setup(self, storage_account, key):
# test chunking functionality by reducing the threshold
# for chunking and the size of each chunk, otherwise
# the tests would take too long to execute
self.bsc = BlobServiceClient(self.account_url(storage_account, "blob"),
credential=key,
max_single_put_size=32 * 1024,
max_block_size=4 * 1024,
max_page_size=4 * 1024,
max_single_get_size=4 * 1024,
transport=AiohttpTestTransport())
self.config = self.bsc._config
self.container_name = self.get_resource_name('utcontainer')
self.blob_types = (BlobType.BlockBlob, BlobType.PageBlob,
BlobType.AppendBlob)
self.bytes = b'Foo'
if self.is_live:
container = self.bsc.get_container_client(self.container_name)
try:
await container.create_container()
except:
pass
def _teardown(self, file_name):
if path.isfile(file_name):
try:
remove(file_name)
except:
pass
def _get_container_reference(self):
return self.get_resource_name(TEST_CONTAINER_PREFIX)
def _get_blob_reference(self, blob_type):
return self.get_resource_name(TEST_BLOB_PREFIXES[blob_type.value])
async def _create_small_blob(self, blob_type):
blob_name = self._get_blob_reference(blob_type)
blob = self.bsc.get_blob_client(self.container_name, blob_name)
await blob.upload_blob(self.bytes, blob_type=blob_type)
return blob
# --Test cases for blob encryption ----------------------------------------
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_missing_attribute_kek_wrap_async(self, resource_group,
location, storage_account,
storage_account_key):
# In the shared method _generate_blob_encryption_key
await self._setup(storage_account, storage_account_key)
self.bsc.require_encryption = True
valid_key = KeyWrapper('key1')
# Act
invalid_key_1 = lambda: None # functions are objects, so this effectively creates an empty object
invalid_key_1.get_key_wrap_algorithm = valid_key.get_key_wrap_algorithm
invalid_key_1.get_kid = valid_key.get_kid
# No attribute wrap_key
self.bsc.key_encryption_key = invalid_key_1
with self.assertRaises(AttributeError):
await self._create_small_blob(BlobType.BlockBlob)
invalid_key_2 = lambda: None # functions are objects, so this effectively creates an empty object
invalid_key_2.wrap_key = valid_key.wrap_key
invalid_key_2.get_kid = valid_key.get_kid
# No attribute get_key_wrap_algorithm
self.bsc.key_encryption_key = invalid_key_2
with self.assertRaises(AttributeError):
await self._create_small_blob(BlobType.BlockBlob)
invalid_key_3 = lambda: None # functions are objects, so this effectively creates an empty object
invalid_key_3.get_key_wrap_algorithm = valid_key.get_key_wrap_algorithm
invalid_key_3.wrap_key = valid_key.wrap_key
# No attribute get_kid
self.bsc.key_encryption_key = invalid_key_2
with self.assertRaises(AttributeError):
await self._create_small_blob(BlobType.BlockBlob)
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_invalid_value_kek_wrap_async(self, resource_group, location,
storage_account,
storage_account_key):
await self._setup(storage_account, storage_account_key)
self.bsc.require_encryption = True
self.bsc.key_encryption_key = KeyWrapper('key1')
self.bsc.key_encryption_key.get_key_wrap_algorithm = None
try:
await self._create_small_blob(BlobType.BlockBlob)
self.fail()
except AttributeError as e:
self.assertEqual(
str(e),
_ERROR_OBJECT_INVALID.format('key encryption key',
'get_key_wrap_algorithm'))
self.bsc.key_encryption_key = KeyWrapper('key1')
self.bsc.key_encryption_key.get_kid = None
with self.assertRaises(AttributeError):
await self._create_small_blob(BlobType.BlockBlob)
self.bsc.key_encryption_key = KeyWrapper('key1')
self.bsc.key_encryption_key.wrap_key = None
with self.assertRaises(AttributeError):
await self._create_small_blob(BlobType.BlockBlob)
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_missing_attribute_kek_unwrap_async(self, resource_group,
location,
storage_account,
storage_account_key):
# Shared between all services in decrypt_blob
await self._setup(storage_account, storage_account_key)
self.bsc.require_encryption = True
valid_key = KeyWrapper('key1')
self.bsc.key_encryption_key = valid_key
blob = await self._create_small_blob(BlobType.BlockBlob)
# Act
# Note that KeyWrapper has a default value for key_id, so these Exceptions
# are not due to non_matching kids.
invalid_key_1 = lambda: None #functions are objects, so this effectively creates an empty object
invalid_key_1.get_kid = valid_key.get_kid
#No attribute unwrap_key
blob.key_encryption_key = invalid_key_1
with self.assertRaises(HttpResponseError):
await (await blob.download_blob()).content_as_bytes()
invalid_key_2 = lambda: None #functions are objects, so this effectively creates an empty object
invalid_key_2.unwrap_key = valid_key.unwrap_key
blob.key_encryption_key = invalid_key_2
#No attribute get_kid
with self.assertRaises(HttpResponseError):
await (await blob.download_blob()).content_as_bytes()
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_invalid_value_kek_unwrap_async(self, resource_group,
location, storage_account,
storage_account_key):
await self._setup(storage_account, storage_account_key)
self.bsc.require_encryption = True
self.bsc.key_encryption_key = KeyWrapper('key1')
blob = await self._create_small_blob(BlobType.BlockBlob)
# Act
blob.key_encryption_key = KeyWrapper('key1')
blob.key_encryption_key.unwrap_key = None
with self.assertRaises(HttpResponseError) as e:
await (await blob.download_blob()).content_as_bytes()
self.assertEqual(str(e.exception), 'Decryption failed.')
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_get_blob_kek_async(self, resource_group, location,
storage_account, storage_account_key):
await self._setup(storage_account, storage_account_key)
self.bsc.require_encryption = True
self.bsc.key_encryption_key = KeyWrapper('key1')
blob = await self._create_small_blob(BlobType.BlockBlob)
# Act
content = await (await blob.download_blob()).content_as_bytes()
# Assert
self.assertEqual(content, self.bytes)
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_get_blob_resolver_async(self, resource_group, location,
storage_account,
storage_account_key):
await self._setup(storage_account, storage_account_key)
self.bsc.require_encryption = True
self.bsc.key_encryption_key = KeyWrapper('key1')
key_resolver = KeyResolver()
key_resolver.put_key(self.bsc.key_encryption_key)
self.bsc.key_resolver_function = key_resolver.resolve_key
blob = await self._create_small_blob(BlobType.BlockBlob)
# Act
self.bsc.key_encryption_key = None
content = await (await blob.download_blob()).content_as_bytes()
# Assert
self.assertEqual(content, self.bytes)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_get_blob_kek_RSA_async(self, resource_group, location,
storage_account,
storage_account_key):
# We can only generate random RSA keys, so this must be run live or
# the playback test will fail due to a change in kek values.
await self._setup(storage_account, storage_account_key)
self.bsc.require_encryption = True
self.bsc.key_encryption_key = RSAKeyWrapper('key2')
blob = await self._create_small_blob(BlobType.BlockBlob)
# Act
content = await blob.download_blob()
data = b""
async for d in content.chunks():
data += d
# Assert
self.assertEqual(data, self.bytes)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_get_blob_nonmatching_kid_async(self, resource_group,
location, storage_account,
storage_account_key):
await self._setup(storage_account, storage_account_key)
self.bsc.require_encryption = True
self.bsc.key_encryption_key = KeyWrapper('key1')
blob = await self._create_small_blob(BlobType.BlockBlob)
# Act
self.bsc.key_encryption_key.kid = 'Invalid'
# Assert
with self.assertRaises(HttpResponseError) as e:
await (await blob.download_blob()).content_as_bytes()
self.assertEqual(str(e.exception), 'Decryption failed.')
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_put_blob_invalid_stream_type_async(self, resource_group,
location,
storage_account,
storage_account_key):
await self._setup(storage_account, storage_account_key)
self.bsc.require_encryption = True
self.bsc.key_encryption_key = KeyWrapper('key1')
small_stream = StringIO(u'small')
large_stream = StringIO(u'large' * self.config.max_single_put_size)
blob_name = self._get_blob_reference(BlobType.BlockBlob)
blob = self.bsc.get_blob_client(self.container_name, blob_name)
# Assert
# Block blob specific single shot
with self.assertRaises(TypeError) as e:
await blob.upload_blob(small_stream, length=5)
self.assertTrue(
'Blob data should be of type bytes.' in str(e.exception))
# Generic blob chunked
with self.assertRaises(TypeError) as e:
await blob.upload_blob(large_stream)
self.assertTrue(
'Blob data should be of type bytes.' in str(e.exception))
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_put_blob_chnking_reqd_mult_of_block_size_async(
self, resource_group, location, storage_account,
storage_account_key):
# parallel tests introduce random order of requests, can only run live
await self._setup(storage_account, storage_account_key)
self.bsc.key_encryption_key = KeyWrapper('key1')
self.bsc.require_encryption = True
content = self.get_random_bytes(self.config.max_single_put_size +
self.config.max_block_size)
blob_name = self._get_blob_reference(BlobType.BlockBlob)
blob = self.bsc.get_blob_client(self.container_name, blob_name)
# Act
await blob.upload_blob(content, max_concurrency=3)
blob_content = await (await blob.download_blob()).content_as_bytes(
max_concurrency=3)
# Assert
self.assertEqual(content, blob_content)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_put_blob_chnking_reqd_non_mult_of_block_size_async(
self, resource_group, location, storage_account,
storage_account_key):
# parallel tests introduce random order of requests, can only run live
await self._setup(storage_account, storage_account_key)
self.bsc.key_encryption_key = KeyWrapper('key1')
self.bsc.require_encryption = True
content = urandom(self.config.max_single_put_size + 1)
blob_name = self._get_blob_reference(BlobType.BlockBlob)
blob = self.bsc.get_blob_client(self.container_name, blob_name)
# Act
await blob.upload_blob(content, max_concurrency=3)
blob_content = await (await blob.download_blob()).content_as_bytes(
max_concurrency=3)
# Assert
self.assertEqual(content, blob_content)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_put_blob_chunking_required_range_specified_async(
self, resource_group, location, storage_account,
storage_account_key):
# parallel tests introduce random order of requests, can only run live
await self._setup(storage_account, storage_account_key)
self.bsc.key_encryption_key = KeyWrapper('key1')
self.bsc.require_encryption = True
content = self.get_random_bytes(self.config.max_single_put_size * 2)
blob_name = self._get_blob_reference(BlobType.BlockBlob)
blob = self.bsc.get_blob_client(self.container_name, blob_name)
# Act
await blob.upload_blob(content,
length=self.config.max_single_put_size + 53,
max_concurrency=3)
blob_content = await (await blob.download_blob()).content_as_bytes(
max_concurrency=3)
# Assert
self.assertEqual(content[:self.config.max_single_put_size + 53],
blob_content)
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_put_block_blob_single_shot_async(self, resource_group,
location, storage_account,
storage_account_key):
await self._setup(storage_account, storage_account_key)
self.bsc.key_encryption_key = KeyWrapper('key1')
self.bsc.require_encryption = True
content = b'small'
blob_name = self._get_blob_reference(BlobType.BlockBlob)
blob = self.bsc.get_blob_client(self.container_name, blob_name)
# Act
await blob.upload_blob(content)
blob_content = await (await blob.download_blob()).content_as_bytes()
# Assert
self.assertEqual(content, blob_content)
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_put_blob_range_async(self, resource_group, location,
storage_account, storage_account_key):
await self._setup(storage_account, storage_account_key)
self.bsc.require_encryption = True
self.bsc.key_encryption_key = KeyWrapper('key1')
content = b'Random repeats' * self.config.max_single_put_size * 5
# All page blob uploads call _upload_chunks, so this will test the ability
# of that function to handle ranges even though it's a small blob
blob_name = self._get_blob_reference(BlobType.BlockBlob)
blob = self.bsc.get_blob_client(self.container_name, blob_name)
# Act
await blob.upload_blob(content[2:],
length=self.config.max_single_put_size + 5,
max_concurrency=1)
blob_content = await (await blob.download_blob()).content_as_bytes(
max_concurrency=1)
# Assert
self.assertEqual(content[2:2 + self.config.max_single_put_size + 5],
blob_content)
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_put_blob_empty_async(self, resource_group, location,
storage_account, storage_account_key):
await self._setup(storage_account, storage_account_key)
self.bsc.key_encryption_key = KeyWrapper('key1')
self.bsc.require_encryption = True
content = b''
blob_name = self._get_blob_reference(BlobType.BlockBlob)
blob = self.bsc.get_blob_client(self.container_name, blob_name)
# Act
await blob.upload_blob(content)
blob_content = await (await blob.download_blob()).content_as_bytes(
max_concurrency=2)
# Assert
self.assertEqual(content, blob_content)
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_put_blob_serial_upload_chunking_async(
self, resource_group, location, storage_account,
storage_account_key):
await self._setup(storage_account, storage_account_key)
self.bsc.key_encryption_key = KeyWrapper('key1')
self.bsc.require_encryption = True
content = self.get_random_bytes(self.config.max_single_put_size + 1)
blob_name = self._get_blob_reference(BlobType.BlockBlob)
blob = self.bsc.get_blob_client(self.container_name, blob_name)
# Act
await blob.upload_blob(content, max_concurrency=1)
blob_content = await (await blob.download_blob()).content_as_bytes(
max_concurrency=1)
# Assert
self.assertEqual(content, blob_content)
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_get_blob_range_beginning_to_middle_async(
self, resource_group, location, storage_account,
storage_account_key):
await self._setup(storage_account, storage_account_key)
self.bsc.key_encryption_key = KeyWrapper('key1')
self.bsc.require_encryption = True
content = self.get_random_bytes(128)
blob_name = self._get_blob_reference(BlobType.BlockBlob)
blob = self.bsc.get_blob_client(self.container_name, blob_name)
# Act
await blob.upload_blob(content, max_concurrency=1)
blob_content = await (await blob.download_blob(
offset=0, length=50)).content_as_bytes(max_concurrency=1)
# Assert
self.assertEqual(content[:50], blob_content)
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_get_blob_range_middle_to_end_async(self, resource_group,
location,
storage_account,
storage_account_key):
await self._setup(storage_account, storage_account_key)
self.bsc.key_encryption_key = KeyWrapper('key1')
self.bsc.require_encryption = True
content = self.get_random_bytes(128)
blob_name = self._get_blob_reference(BlobType.BlockBlob)
blob = self.bsc.get_blob_client(self.container_name, blob_name)
# Act
await blob.upload_blob(content, max_concurrency=1)
blob_content = await (await blob.download_blob(
offset=100, length=28)).content_as_bytes()
blob_content2 = await (await blob.download_blob(offset=100
)).content_as_bytes()
# Assert
self.assertEqual(content[100:], blob_content)
self.assertEqual(content[100:], blob_content2)
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_get_blob_range_middle_to_middle_async(
self, resource_group, location, storage_account,
storage_account_key):
await self._setup(storage_account, storage_account_key)
self.bsc.key_encryption_key = KeyWrapper('key1')
self.bsc.require_encryption = True
content = self.get_random_bytes(128)
blob_name = self._get_blob_reference(BlobType.BlockBlob)
blob = self.bsc.get_blob_client(self.container_name, blob_name)
# Act
await blob.upload_blob(content)
blob_content = await (await blob.download_blob(
offset=5, length=93)).content_as_bytes()
# Assert
self.assertEqual(content[5:98], blob_content)
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_get_blob_range_aligns_on_16_byte_block_async(
self, resource_group, location, storage_account,
storage_account_key):
await self._setup(storage_account, storage_account_key)
self.bsc.key_encryption_key = KeyWrapper('key1')
self.bsc.require_encryption = True
content = self.get_random_bytes(128)
blob_name = self._get_blob_reference(BlobType.BlockBlob)
blob = self.bsc.get_blob_client(self.container_name, blob_name)
# Act
await blob.upload_blob(content)
blob_content = await (await blob.download_blob(
offset=48, length=16)).content_as_bytes()
# Assert
self.assertEqual(content[48:64], blob_content)
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_get_blob_range_expnded_to_begin_bloc_align_async(
self, resource_group, location, storage_account,
storage_account_key):
await self._setup(storage_account, storage_account_key)
self.bsc.key_encryption_key = KeyWrapper('key1')
self.bsc.require_encryption = True
content = self.get_random_bytes(128)
blob_name = self._get_blob_reference(BlobType.BlockBlob)
blob = self.bsc.get_blob_client(self.container_name, blob_name)
# Act
await blob.upload_blob(content)
blob_content = await (await blob.download_blob(
offset=5, length=50)).content_as_bytes()
# Assert
self.assertEqual(content[5:55], blob_content)
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_get_blob_range_expanded_to_beginning_iv_async(
self, resource_group, location, storage_account,
storage_account_key):
await self._setup(storage_account, storage_account_key)
self.bsc.key_encryption_key = KeyWrapper('key1')
self.bsc.require_encryption = True
content = self.get_random_bytes(128)
blob_name = self._get_blob_reference(BlobType.BlockBlob)
blob = self.bsc.get_blob_client(self.container_name, blob_name)
# Act
await blob.upload_blob(content)
blob_content = await (await blob.download_blob(
offset=22, length=20)).content_as_bytes()
# Assert
self.assertEqual(content[22:42], blob_content)
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_put_blob_strict_mode_async(self, resource_group, location,
storage_account,
storage_account_key):
await self._setup(storage_account, storage_account_key)
self.bsc.require_encryption = True
content = urandom(512)
# Assert
for service in self.blob_types:
blob_name = self._get_blob_reference(service)
blob = self.bsc.get_blob_client(self.container_name, blob_name)
with self.assertRaises(ValueError):
await blob.upload_blob(content, blob_type=service)
stream = BytesIO(content)
with self.assertRaises(ValueError):
await blob.upload_blob(stream, length=512, blob_type=service)
file_name = 'strict_mode_async.temp.dat'
with open(file_name, 'wb') as stream:
stream.write(content)
with open(file_name, 'rb') as stream:
with self.assertRaises(ValueError):
await blob.upload_blob(stream, blob_type=service)
with self.assertRaises(ValueError):
await blob.upload_blob('To encrypt', blob_type=service)
self._teardown(file_name)
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_get_blob_strict_mode_no_policy_async(
self, resource_group, location, storage_account,
storage_account_key):
await self._setup(storage_account, storage_account_key)
self.bsc.require_encryption = True
self.bsc.key_encryption_key = KeyWrapper('key1')
blob = await self._create_small_blob(BlobType.BlockBlob)
# Act
blob.key_encryption_key = None
# Assert
with self.assertRaises(ValueError):
await (await blob.download_blob()).content_as_bytes()
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_get_blob_strict_mode_unencrypted_blob_async(
self, resource_group, location, storage_account,
storage_account_key):
await self._setup(storage_account, storage_account_key)
blob = await self._create_small_blob(BlobType.BlockBlob)
# Act
blob.require_encryption = True
blob.key_encryption_key = KeyWrapper('key1')
# Assert
with self.assertRaises(HttpResponseError):
await (await blob.download_blob()).content_as_bytes()
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_invalid_methods_fail_block_async(self, resource_group,
location, storage_account,
storage_account_key):
await self._setup(storage_account, storage_account_key)
self.bsc.key_encryption_key = KeyWrapper('key1')
blob_name = self._get_blob_reference(BlobType.BlockBlob)
blob = self.bsc.get_blob_client(self.container_name, blob_name)
# Assert
with self.assertRaises(ValueError) as e:
await blob.stage_block('block1', urandom(32))
self.assertEqual(str(e.exception),
_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION)
with self.assertRaises(ValueError) as e:
await blob.commit_block_list(['block1'])
self.assertEqual(str(e.exception),
_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION)
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_invalid_methods_fail_append_async(self, resource_group,
location, storage_account,
storage_account_key):
await self._setup(storage_account, storage_account_key)
self.bsc.key_encryption_key = KeyWrapper('key1')
blob_name = self._get_blob_reference(BlobType.AppendBlob)
blob = self.bsc.get_blob_client(self.container_name, blob_name)
# Assert
with self.assertRaises(ValueError) as e:
await blob.append_block(urandom(32))
self.assertEqual(str(e.exception),
_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION)
with self.assertRaises(ValueError) as e:
await blob.create_append_blob()
self.assertEqual(str(e.exception),
_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION)
# All append_from operations funnel into append_from_stream, so testing one is sufficient
with self.assertRaises(ValueError) as e:
await blob.upload_blob(b'To encrypt',
blob_type=BlobType.AppendBlob)
self.assertEqual(str(e.exception),
_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION)
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_invalid_methods_fail_page_async(self, resource_group,
location, storage_account,
storage_account_key):
await self._setup(storage_account, storage_account_key)
self.bsc.key_encryption_key = KeyWrapper('key1')
blob_name = self._get_blob_reference(BlobType.PageBlob)
blob = self.bsc.get_blob_client(self.container_name, blob_name)
# Assert
with self.assertRaises(ValueError) as e:
await blob.upload_page(urandom(512),
offset=0,
length=512,
blob_type=BlobType.PageBlob)
self.assertEqual(str(e.exception),
_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION)
with self.assertRaises(ValueError) as e:
await blob.create_page_blob(512)
self.assertEqual(str(e.exception),
_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION)
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_validate_encryption_async(self, resource_group, location,
storage_account,
storage_account_key):
await self._setup(storage_account, storage_account_key)
self.bsc.require_encryption = True
kek = KeyWrapper('key1')
self.bsc.key_encryption_key = kek
blob = await self._create_small_blob(BlobType.BlockBlob)
# Act
blob.require_encryption = False
blob.key_encryption_key = None
content = await blob.download_blob()
data = await content.content_as_bytes()
encryption_data = _dict_to_encryption_data(
loads(content.properties.metadata['encryptiondata']))
iv = encryption_data.content_encryption_IV
content_encryption_key = _validate_and_unwrap_cek(
encryption_data, kek, None)
cipher = _generate_AES_CBC_cipher(content_encryption_key, iv)
decryptor = cipher.decryptor()
unpadder = PKCS7(128).unpadder()
content = decryptor.update(data) + decryptor.finalize()
content = unpadder.update(content) + unpadder.finalize()
self.assertEqual(self.bytes, content)
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_create_block_blob_from_star_async(self, resource_group,
location, storage_account,
storage_account_key):
await self._setup(storage_account, storage_account_key)
await self._create_blob_from_star(BlobType.BlockBlob, "blob1",
self.bytes, self.bytes)
stream = BytesIO(self.bytes)
await self._create_blob_from_star(BlobType.BlockBlob, "blob2",
self.bytes, stream)
file_name = 'block_star_async.temp.dat'
with open(file_name, 'wb') as stream:
stream.write(self.bytes)
with open(file_name, 'rb') as stream:
await self._create_blob_from_star(BlobType.BlockBlob, "blob3",
self.bytes, stream)
await self._create_blob_from_star(BlobType.BlockBlob, "blob4",
b'To encrypt', 'To encrypt')
self._teardown(file_name)
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_create_page_blob_from_star_async(self, resource_group,
location, storage_account,
storage_account_key):
await self._setup(storage_account, storage_account_key)
content = self.get_random_bytes(512)
await self._create_blob_from_star(BlobType.PageBlob, "blob1", content,
content)
stream = BytesIO(content)
await self._create_blob_from_star(BlobType.PageBlob,
"blob2",
content,
stream,
length=512)
file_name = 'page_star_async.temp.dat'
with open(file_name, 'wb') as stream:
stream.write(content)
with open(file_name, 'rb') as stream:
await self._create_blob_from_star(BlobType.PageBlob, "blob3",
content, stream)
self._teardown(file_name)
async def _create_blob_from_star(self, blob_type, blob_name, content, data,
**kwargs):
blob = self.bsc.get_blob_client(self.container_name, blob_name)
blob.key_encryption_key = KeyWrapper('key1')
blob.require_encryption = True
await blob.upload_blob(data, blob_type=blob_type, **kwargs)
blob_content = await (await blob.download_blob()).content_as_bytes()
self.assertEqual(content, blob_content)
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_get_blob_to_star_async(self, resource_group, location,
storage_account,
storage_account_key):
await self._setup(storage_account, storage_account_key)
self.bsc.require_encryption = True
self.bsc.key_encryption_key = KeyWrapper('key1')
blob = await self._create_small_blob(BlobType.BlockBlob)
# Act
content = await blob.download_blob()
iter_blob = b""
async for data in content.chunks():
iter_blob += data
bytes_blob = await (await blob.download_blob()).content_as_bytes()
stream_blob = BytesIO()
await (await blob.download_blob()).download_to_stream(stream_blob)
stream_blob.seek(0)
text_blob = await (await blob.download_blob()).content_as_text()
# Assert
self.assertEqual(self.bytes, iter_blob)
self.assertEqual(self.bytes, bytes_blob)
self.assertEqual(self.bytes, stream_blob.read())
self.assertEqual(self.bytes.decode(), text_blob)
class StorageGetBlobTestAsync(AsyncStorageTestCase):
# --Helpers-----------------------------------------------------------------
async def _setup(self, name, key):
self.bsc = BlobServiceClient(self.account_url(name, "blob"),
credential=key,
max_single_get_size=32 * 1024,
max_chunk_get_size=4 * 1024,
transport=AiohttpTestTransport())
self.config = self.bsc._config
self.container_name = self.get_resource_name('utcontainer')
self.byte_blob = self.get_resource_name('byteblob')
self.byte_data = self.get_random_bytes(64 * 1024 + 5)
if self.is_live:
container = self.bsc.get_container_client(self.container_name)
try:
await container.create_container()
except:
pass
blob = self.bsc.get_blob_client(self.container_name,
self.byte_blob)
await blob.upload_blob(self.byte_data, overwrite=True)
def _teardown(self, file_name):
if path.isfile(file_name):
try:
remove(file_name)
except:
pass
def _get_blob_reference(self):
return self.get_resource_name(TEST_BLOB_PREFIX)
class NonSeekableFile(object):
def __init__(self, wrapped_file):
self.wrapped_file = wrapped_file
def write(self, data):
self.wrapped_file.write(data)
def read(self, count):
return self.wrapped_file.read(count)
def seekable(self):
return False
# -- Get test cases for blobs ----------------------------------------------
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_unicode_get_blob_unicode_data_async(self, resource_group,
location,
storage_account,
storage_account_key):
# Arrange
await self._setup(storage_account.name, storage_account_key)
blob_data = u'hello world啊齄丂狛狜'.encode('utf-8')
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
await blob.upload_blob(blob_data)
# Act
content = await blob.download_blob()
# Assert
self.assertIsInstance(content.properties, BlobProperties)
self.assertEqual(await content.readall(), blob_data)
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_unicode_get_blob_binary_data_async(self, resource_group,
location,
storage_account,
storage_account_key):
# Arrange
await self._setup(storage_account.name, storage_account_key)
base64_data = 'AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygpKissLS4vMDEyMzQ1Njc4OTo7PD0+P0BBQkNERUZHSElKS0xNTk9QUVJTVFVWV1hZWltcXV5fYGFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6e3x9fn+AgYKDhIWGh4iJiouMjY6PkJGSk5SVlpeYmZqbnJ2en6ChoqOkpaanqKmqq6ytrq+wsbKztLW2t7i5uru8vb6/wMHCw8TFxsfIycrLzM3Oz9DR0tPU1dbX2Nna29zd3t/g4eLj5OXm5+jp6uvs7e7v8PHy8/T19vf4+fr7/P3+/wABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fICEiIyQlJicoKSorLC0uLzAxMjM0NTY3ODk6Ozw9Pj9AQUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVpbXF1eX2BhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ent8fX5/gIGCg4SFhoeIiYqLjI2Oj5CRkpOUlZaXmJmam5ydnp+goaKjpKWmp6ipqqusra6vsLGys7S1tre4ubq7vL2+v8DBwsPExcbHyMnKy8zNzs/Q0dLT1NXW19jZ2tvc3d7f4OHi4+Tl5ufo6err7O3u7/Dx8vP09fb3+Pn6+/z9/v8AAQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyAhIiMkJSYnKCkqKywtLi8wMTIzNDU2Nzg5Ojs8PT4/QEFCQ0RFRkdISUpLTE1OT1BRUlNUVVZXWFlaW1xdXl9gYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXp7fH1+f4CBgoOEhYaHiImKi4yNjo+QkZKTlJWWl5iZmpucnZ6foKGio6SlpqeoqaqrrK2ur7CxsrO0tba3uLm6u7y9vr/AwcLDxMXGx8jJysvMzc7P0NHS09TV1tfY2drb3N3e3+Dh4uPk5ebn6Onq6+zt7u/w8fLz9PX29/j5+vv8/f7/AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygpKissLS4vMDEyMzQ1Njc4OTo7PD0+P0BBQkNERUZHSElKS0xNTk9QUVJTVFVWV1hZWltcXV5fYGFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6e3x9fn+AgYKDhIWGh4iJiouMjY6PkJGSk5SVlpeYmZqbnJ2en6ChoqOkpaanqKmqq6ytrq+wsbKztLW2t7i5uru8vb6/wMHCw8TFxsfIycrLzM3Oz9DR0tPU1dbX2Nna29zd3t/g4eLj5OXm5+jp6uvs7e7v8PHy8/T19vf4+fr7/P3+/w=='
binary_data = base64.b64decode(base64_data)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
await blob.upload_blob(binary_data)
# Act
content = await blob.download_blob()
# Assert
self.assertIsInstance(content.properties, BlobProperties)
self.assertEqual(await content.readall(), binary_data)
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_get_blob_no_content_async(self, resource_group, location,
storage_account,
storage_account_key):
# Arrange
await self._setup(storage_account.name, storage_account_key)
blob_data = b''
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
await blob.upload_blob(blob_data)
# Act
content = await blob.download_blob()
# Assert
self.assertEqual(blob_data, await content.readall())
self.assertEqual(0, content.properties.size)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_get_blob_to_bytes_async(self, resource_group, location,
storage_account,
storage_account_key):
# parallel tests introduce random order of requests, can only run live
# Arrange
await self._setup(storage_account.name, storage_account_key)
blob = self.bsc.get_blob_client(self.container_name, self.byte_blob)
# Act
content = await (await blob.download_blob(max_concurrency=2)).readall()
# Assert
self.assertEqual(self.byte_data, content)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_ranged_get_blob_to_bytes_with_single_byte_async(
self, resource_group, location, storage_account,
storage_account_key):
# parallel tests introduce random order of requests, can only run live
# Arrange
await self._setup(storage_account.name, storage_account_key)
blob = self.bsc.get_blob_client(self.container_name, self.byte_blob)
# Act
content = await (await blob.download_blob(offset=0,
length=1)).readall()
# Assert
self.assertEqual(1, len(content))
self.assertEqual(self.byte_data[0], content[0])
# Act
content = await (await blob.download_blob(offset=5,
length=1)).readall()
# Assert
self.assertEqual(1, len(content))
self.assertEqual(self.byte_data[5], content[0])
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_ranged_get_blob_to_bytes_with_zero_byte_async(
self, resource_group, location, storage_account,
storage_account_key):
await self._setup(storage_account.name, storage_account_key)
blob_data = b''
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
await blob.upload_blob(blob_data)
# Act
# the get request should fail in this case since the blob is empty and yet there is a range specified
with self.assertRaises(HttpResponseError) as e:
await blob.download_blob(offset=0, length=5)
self.assertEqual(StorageErrorCode.invalid_range,
e.exception.error_code)
with self.assertRaises(HttpResponseError) as e:
await blob.download_blob(offset=3, length=5)
self.assertEqual(StorageErrorCode.invalid_range,
e.exception.error_code)
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_ranged_get_blob_with_missing_start_range_async(
self, resource_group, location, storage_account,
storage_account_key):
await self._setup(storage_account.name, storage_account_key)
blob_data = b'foobar'
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
await blob.upload_blob(blob_data)
# Act
# the get request should fail fast in this case since start_range is missing while end_range is specified
with self.assertRaises(ValueError):
await blob.download_blob(length=3)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_get_blob_to_bytes_snapshot_async(self, resource_group,
location, storage_account,
storage_account_key):
# parallel tests introduce random order of requests, can only run live
# Arrange
await self._setup(storage_account.name, storage_account_key)
blob = self.bsc.get_blob_client(self.container_name, self.byte_blob)
snapshot_ref = await blob.create_snapshot()
snapshot = self.bsc.get_blob_client(self.container_name,
self.byte_blob,
snapshot=snapshot_ref)
await blob.upload_blob(
self.byte_data,
overwrite=True) # Modify the blob so the Etag no longer matches
# Act
content = await (await
snapshot.download_blob(max_concurrency=2)).readall()
# Assert
self.assertEqual(self.byte_data, content)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_get_blob_to_bytes_with_progress_async(
self, resource_group, location, storage_account,
storage_account_key):
# parallel tests introduce random order of requests, can only run live
# Arrange
await self._setup(storage_account.name, storage_account_key)
progress = []
blob = self.bsc.get_blob_client(self.container_name, self.byte_blob)
def callback(response):
current = response.context['download_stream_current']
total = response.context['data_stream_total']
progress.append((current, total))
# Act
content = await (await
blob.download_blob(raw_response_hook=callback,
max_concurrency=2)).readall()
# Assert
self.assertEqual(self.byte_data, content)
self.assert_download_progress(len(self.byte_data),
self.config.max_chunk_get_size,
self.config.max_single_get_size,
progress)
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_get_blob_to_bytes_non_parallel_async(
self, resource_group, location, storage_account,
storage_account_key):
# Arrange
await self._setup(storage_account.name, storage_account_key)
progress = []
blob = self.bsc.get_blob_client(self.container_name, self.byte_blob)
def callback(response):
current = response.context['download_stream_current']
total = response.context['data_stream_total']
progress.append((current, total))
# Act
content = await (await
blob.download_blob(raw_response_hook=callback,
max_concurrency=1)).readall()
# Assert
self.assertEqual(self.byte_data, content)
self.assert_download_progress(len(self.byte_data),
self.config.max_chunk_get_size,
self.config.max_single_get_size,
progress)
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_get_blob_to_bytes_small_async(self, resource_group,
location, storage_account,
storage_account_key):
# Arrange
await self._setup(storage_account.name, storage_account_key)
blob_data = self.get_random_bytes(1024)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
await blob.upload_blob(blob_data)
progress = []
def callback(response):
current = response.context['download_stream_current']
total = response.context['data_stream_total']
progress.append((current, total))
# Act
content = await (await blob.download_blob(raw_response_hook=callback
)).readall()
# Assert
self.assertEqual(blob_data, content)
self.assert_download_progress(len(blob_data),
self.config.max_chunk_get_size,
self.config.max_single_get_size,
progress)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_get_blob_to_stream_async(self, resource_group, location,
storage_account,
storage_account_key):
# parallel tests introduce random order of requests, can only run live
# Arrange
await self._setup(storage_account.name, storage_account_key)
blob = self.bsc.get_blob_client(self.container_name, self.byte_blob)
# Act
FILE_PATH = 'get_blob_to_stream_async.temp.dat'
with open(FILE_PATH, 'wb') as stream:
downloader = await blob.download_blob(max_concurrency=2)
read_bytes = await downloader.readinto(stream)
# Assert
self.assertEqual(read_bytes, len(self.byte_data))
with open(FILE_PATH, 'rb') as stream:
actual = stream.read()
self.assertEqual(self.byte_data, actual)
self._teardown(FILE_PATH)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_get_blob_to_stream_with_progress_async(
self, resource_group, location, storage_account,
storage_account_key):
# parallel tests introduce random order of requests, can only run live
# Arrange
await self._setup(storage_account.name, storage_account_key)
progress = []
blob = self.bsc.get_blob_client(self.container_name, self.byte_blob)
def callback(response):
current = response.context['download_stream_current']
total = response.context['data_stream_total']
progress.append((current, total))
# Act
FILE_PATH = 'blob_to_stream_with_progress_async.temp.dat'
with open(FILE_PATH, 'wb') as stream:
downloader = await blob.download_blob(raw_response_hook=callback,
max_concurrency=2)
read_bytes = await downloader.readinto(stream)
# Assert
self.assertEqual(read_bytes, len(self.byte_data))
with open(FILE_PATH, 'rb') as stream:
actual = stream.read()
self.assertEqual(self.byte_data, actual)
self.assert_download_progress(len(self.byte_data),
self.config.max_chunk_get_size,
self.config.max_single_get_size,
progress)
self._teardown(FILE_PATH)
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_get_blob_to_stream_non_parallel_async(
self, resource_group, location, storage_account,
storage_account_key):
# Arrange
await self._setup(storage_account.name, storage_account_key)
progress = []
blob = self.bsc.get_blob_client(self.container_name, self.byte_blob)
def callback(response):
current = response.context['download_stream_current']
total = response.context['data_stream_total']
progress.append((current, total))
# Act
FILE_PATH = 'blob_to_stream_non_parallel_async.temp.dat'
with open(FILE_PATH, 'wb') as stream:
downloader = await blob.download_blob(raw_response_hook=callback,
max_concurrency=1)
read_bytes = await downloader.readinto(stream)
# Assert
self.assertEqual(read_bytes, len(self.byte_data))
with open(FILE_PATH, 'rb') as stream:
actual = stream.read()
self.assertEqual(self.byte_data, actual)
self.assert_download_progress(len(self.byte_data),
self.config.max_chunk_get_size,
self.config.max_single_get_size,
progress)
self._teardown(FILE_PATH)
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_get_blob_to_stream_small_async(self, resource_group,
location, storage_account,
storage_account_key):
# Arrange
await self._setup(storage_account.name, storage_account_key)
blob_data = self.get_random_bytes(1024)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
await blob.upload_blob(blob_data)
progress = []
def callback(response):
current = response.context['download_stream_current']
total = response.context['data_stream_total']
progress.append((current, total))
# Act
FILE_PATH = 'blob_to_stream_small_async.temp.dat'
with open(FILE_PATH, 'wb') as stream:
downloader = await blob.download_blob(raw_response_hook=callback,
max_concurrency=2)
read_bytes = await downloader.readinto(stream)
# Assert
self.assertEqual(read_bytes, 1024)
with open(FILE_PATH, 'rb') as stream:
actual = stream.read()
self.assertEqual(blob_data, actual)
self.assert_download_progress(len(blob_data),
self.config.max_chunk_get_size,
self.config.max_single_get_size,
progress)
self._teardown(FILE_PATH)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_ranged_get_blob_to_path_async(self, resource_group,
location, storage_account,
storage_account_key):
# parallel tests introduce random order of requests, can only run live
# Arrange
await self._setup(storage_account.name, storage_account_key)
blob = self.bsc.get_blob_client(self.container_name, self.byte_blob)
# Act
end_range = self.config.max_single_get_size
FILE_PATH = 'ranged_get_blob_to_path_async.temp.dat'
with open(FILE_PATH, 'wb') as stream:
downloader = await blob.download_blob(offset=1,
length=end_range - 1,
max_concurrency=2)
read_bytes = await downloader.readinto(stream)
# Assert
self.assertEqual(read_bytes, end_range - 1)
with open(FILE_PATH, 'rb') as stream:
actual = stream.read()
self.assertEqual(self.byte_data[1:end_range], actual)
self._teardown(FILE_PATH)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_ranged_get_blob_to_path_with_progress_async(
self, resource_group, location, storage_account,
storage_account_key):
# parallel tests introduce random order of requests, can only run live
# Arrange
await self._setup(storage_account.name, storage_account_key)
progress = []
blob = self.bsc.get_blob_client(self.container_name, self.byte_blob)
def callback(response):
current = response.context['download_stream_current']
total = response.context['data_stream_total']
progress.append((current, total))
# Act
start_range = 3
end_range = self.config.max_single_get_size + 1024
FILE_PATH = 'get_blob_to_path_with_progress_async.temp.dat'
with open(FILE_PATH, 'wb') as stream:
downloader = await blob.download_blob(offset=start_range,
length=end_range,
raw_response_hook=callback,
max_concurrency=2)
read_bytes = await downloader.readinto(stream)
# Assert
self.assertEqual(read_bytes, self.config.max_single_get_size + 1024)
with open(FILE_PATH, 'rb') as stream:
actual = stream.read()
self.assertEqual(
self.byte_data[start_range:end_range + start_range], actual)
self.assert_download_progress(end_range,
self.config.max_chunk_get_size,
self.config.max_single_get_size,
progress)
self._teardown(FILE_PATH)
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_ranged_get_blob_to_path_small_async(self, resource_group,
location,
storage_account,
storage_account_key):
# Arrange
await self._setup(storage_account.name, storage_account_key)
blob = self.bsc.get_blob_client(self.container_name, self.byte_blob)
# Act
FILE_PATH = 'get_blob_to_path_small_asyncc.temp.dat'
with open(FILE_PATH, 'wb') as stream:
downloader = await blob.download_blob(offset=1,
length=4,
max_concurrency=2)
read_bytes = await downloader.readinto(stream)
# Assert
self.assertEqual(read_bytes, 4)
with open(FILE_PATH, 'rb') as stream:
actual = stream.read()
self.assertEqual(self.byte_data[1:5], actual)
self._teardown(FILE_PATH)
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_ranged_get_blob_to_path_non_parallel_async(
self, resource_group, location, storage_account,
storage_account_key):
# Arrange
await self._setup(storage_account.name, storage_account_key)
blob = self.bsc.get_blob_client(self.container_name, self.byte_blob)
# Act
FILE_PATH = 'granged_get_blob_to_path_non_parallel_async.temp.dat'
with open(FILE_PATH, 'wb') as stream:
downloader = await blob.download_blob(offset=1,
length=3,
max_concurrency=1)
read_bytes = await downloader.readinto(stream)
# Assert
self.assertEqual(read_bytes, 3)
with open(FILE_PATH, 'rb') as stream:
actual = stream.read()
self.assertEqual(self.byte_data[1:4], actual)
self._teardown(FILE_PATH)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_ranged_get_blob_to_path_invalid_range_parallel_async(
self, resource_group, location, storage_account,
storage_account_key):
# parallel tests introduce random order of requests, can only run live
# Arrange
await self._setup(storage_account.name, storage_account_key)
blob_size = self.config.max_single_get_size + 1
blob_data = self.get_random_bytes(blob_size)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
await blob.upload_blob(blob_data)
# Act
FILE_PATH = 'path_invalid_range_parallel_async.temp.dat'
end_range = 2 * self.config.max_single_get_size
with open(FILE_PATH, 'wb') as stream:
downloader = await blob.download_blob(offset=1,
length=end_range,
max_concurrency=2)
read_bytes = await downloader.readinto(stream)
# Assert
self.assertEqual(read_bytes, blob_size)
with open(FILE_PATH, 'rb') as stream:
actual = stream.read()
self.assertEqual(blob_data[1:blob_size], actual)
self._teardown(FILE_PATH)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_ranged_get_blob_to_path_invalid_range_non_parallel_async(
self, resource_group, location, storage_account,
storage_account_key):
# parallel tests introduce random order of requests, can only run live
# Arrange
await self._setup(storage_account.name, storage_account_key)
blob_size = 1024
blob_data = self.get_random_bytes(blob_size)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
await blob.upload_blob(blob_data)
# Act
end_range = 2 * self.config.max_single_get_size
FILE_PATH = 'path_invalid_range_non_parallel_asy.temp.dat'
with open(FILE_PATH, 'wb') as stream:
downloader = await blob.download_blob(offset=1,
length=end_range,
max_concurrency=2)
read_bytes = await downloader.readinto(stream)
# Assert
self.assertEqual(read_bytes, blob_size)
with open(FILE_PATH, 'rb') as stream:
actual = stream.read()
self.assertEqual(blob_data[1:blob_size], actual)
self._teardown(FILE_PATH)
# Assert
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_get_blob_to_text_async(self, resource_group, location,
storage_account,
storage_account_key):
# parallel tests introduce random order of requests, can only run live
# Arrange
await self._setup(storage_account.name, storage_account_key)
text_blob = self.get_resource_name('textblob')
text_data = self.get_random_text_data(self.config.max_single_get_size +
1)
blob = self.bsc.get_blob_client(self.container_name, text_blob)
await blob.upload_blob(text_data)
# Act
stream = await blob.download_blob(max_concurrency=2, encoding='UTF-8')
content = await stream.readall()
# Assert
self.assertEqual(text_data, content)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_get_blob_to_text_with_progress_async(
self, resource_group, location, storage_account,
storage_account_key):
# parallel tests introduce random order of requests, can only run live
# Arrange
await self._setup(storage_account.name, storage_account_key)
text_blob = self.get_resource_name('textblob')
text_data = self.get_random_text_data(self.config.max_single_get_size +
1)
blob = self.bsc.get_blob_client(self.container_name, text_blob)
await blob.upload_blob(text_data)
progress = []
def callback(response):
current = response.context['download_stream_current']
total = response.context['data_stream_total']
progress.append((current, total))
# Act
stream = await blob.download_blob(raw_response_hook=callback,
max_concurrency=2,
encoding='UTF-8')
content = await stream.readall()
# Assert
self.assertEqual(text_data, content)
self.assert_download_progress(len(text_data.encode('utf-8')),
self.config.max_chunk_get_size,
self.config.max_single_get_size,
progress)
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_get_blob_to_text_non_parallel_async(self, resource_group,
location,
storage_account,
storage_account_key):
# Arrange
await self._setup(storage_account.name, storage_account_key)
text_blob = self._get_blob_reference()
text_data = self.get_random_text_data(self.config.max_single_get_size +
1)
blob = self.bsc.get_blob_client(self.container_name, text_blob)
await blob.upload_blob(text_data)
progress = []
def callback(response):
current = response.context['download_stream_current']
total = response.context['data_stream_total']
progress.append((current, total))
# Act
stream = await blob.download_blob(raw_response_hook=callback,
max_concurrency=1,
encoding='UTF-8')
content = await stream.readall()
# Assert
self.assertEqual(text_data, content)
self.assert_download_progress(len(text_data),
self.config.max_chunk_get_size,
self.config.max_single_get_size,
progress)
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_get_blob_to_text_small_async(self, resource_group, location,
storage_account,
storage_account_key):
# Arrange
await self._setup(storage_account.name, storage_account_key)
blob_data = self.get_random_text_data(1024)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
await blob.upload_blob(blob_data)
progress = []
def callback(response):
current = response.context['download_stream_current']
total = response.context['data_stream_total']
progress.append((current, total))
# Act
stream = await blob.download_blob(raw_response_hook=callback,
encoding='UTF-8')
content = await stream.readall()
# Assert
self.assertEqual(blob_data, content)
self.assert_download_progress(len(blob_data),
self.config.max_chunk_get_size,
self.config.max_single_get_size,
progress)
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_get_blob_to_text_with_encoding_async(
self, resource_group, location, storage_account,
storage_account_key):
# Arrange
await self._setup(storage_account.name, storage_account_key)
text = u'hello 啊齄丂狛狜 world'
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
await blob.upload_blob(text, encoding='utf-16')
# Act
stream = await blob.download_blob(encoding='utf-16')
content = await stream.readall()
# Assert
self.assertEqual(text, content)
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_get_blob_to_text_with_encoding_and_progress_async(
self, resource_group, location, storage_account,
storage_account_key):
# Arrange
await self._setup(storage_account.name, storage_account_key)
text = u'hello 啊齄丂狛狜 world'
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
await blob.upload_blob(text, encoding='utf-16')
# Act
progress = []
def callback(response):
current = response.context['download_stream_current']
total = response.context['data_stream_total']
progress.append((current, total))
stream = await blob.download_blob(raw_response_hook=callback,
encoding='utf-16')
content = await stream.readall()
# Assert
self.assertEqual(text, content)
self.assert_download_progress(len(text.encode('utf-8')),
self.config.max_chunk_get_size,
self.config.max_single_get_size,
progress)
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_get_blob_non_seekable_async(self, resource_group, location,
storage_account,
storage_account_key):
# Arrange
await self._setup(storage_account.name, storage_account_key)
blob = self.bsc.get_blob_client(self.container_name, self.byte_blob)
# Act
FILE_PATH = 'get_blob_non_seekable_async.temp.dat'
with open(FILE_PATH, 'wb') as stream:
non_seekable_stream = StorageGetBlobTestAsync.NonSeekableFile(
stream)
downloader = await blob.download_blob(max_concurrency=1)
read_bytes = await downloader.readinto(non_seekable_stream)
# Assert
self.assertEqual(read_bytes, len(self.byte_data))
with open(FILE_PATH, 'rb') as stream:
actual = stream.read()
self.assertEqual(self.byte_data, actual)
self._teardown(FILE_PATH)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_get_blob_non_seekable_parallel_async(
self, resource_group, location, storage_account,
storage_account_key):
# parallel tests introduce random order of requests, can only run live
# Arrange
await self._setup(storage_account.name, storage_account_key)
blob = self.bsc.get_blob_client(self.container_name, self.byte_blob)
# Act
FILE_PATH = 'et_blob_non_seekable_parallel_asyn.temp.dat'
with open(FILE_PATH, 'wb') as stream:
non_seekable_stream = StorageGetBlobTestAsync.NonSeekableFile(
stream)
with self.assertRaises(ValueError):
downloader = await blob.download_blob(max_concurrency=2)
properties = await downloader.readinto(non_seekable_stream)
self._teardown(FILE_PATH)
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_get_blob_to_stream_exact_get_size_async(
self, resource_group, location, storage_account,
storage_account_key):
# Arrange
await self._setup(storage_account.name, storage_account_key)
blob_name = self._get_blob_reference()
byte_data = self.get_random_bytes(self.config.max_single_get_size)
blob = self.bsc.get_blob_client(self.container_name, blob_name)
await blob.upload_blob(byte_data)
progress = []
def callback(response):
current = response.context['download_stream_current']
total = response.context['data_stream_total']
progress.append((current, total))
# Act
FILE_PATH = 'stream_exact_get_size_async.temp.dat'
with open(FILE_PATH, 'wb') as stream:
downloader = await blob.download_blob(raw_response_hook=callback,
max_concurrency=2)
properties = await downloader.readinto(stream)
# Assert
with open(FILE_PATH, 'rb') as stream:
actual = stream.read()
self.assertEqual(byte_data, actual)
self.assert_download_progress(len(byte_data),
self.config.max_chunk_get_size,
self.config.max_single_get_size,
progress)
self._teardown(FILE_PATH)
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_get_blob_exact_get_size_async(self, resource_group,
location, storage_account,
storage_account_key):
# Arrange
await self._setup(storage_account.name, storage_account_key)
blob_name = self._get_blob_reference()
byte_data = self.get_random_bytes(self.config.max_single_get_size)
blob = self.bsc.get_blob_client(self.container_name, blob_name)
await blob.upload_blob(byte_data)
progress = []
def callback(response):
current = response.context['download_stream_current']
total = response.context['data_stream_total']
progress.append((current, total))
# Act
content = await (await blob.download_blob(raw_response_hook=callback
)).readall()
# Assert
self.assertEqual(byte_data, content)
self.assert_download_progress(len(byte_data),
self.config.max_chunk_get_size,
self.config.max_single_get_size,
progress)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_get_blob_exact_chunk_size_async(self, resource_group,
location, storage_account,
storage_account_key):
# parallel tests introduce random order of requests, can only run live
# Arrange
await self._setup(storage_account.name, storage_account_key)
blob_name = self._get_blob_reference()
byte_data = self.get_random_bytes(self.config.max_single_get_size +
self.config.max_chunk_get_size)
blob = self.bsc.get_blob_client(self.container_name, blob_name)
await blob.upload_blob(byte_data)
progress = []
def callback(response):
current = response.context['download_stream_current']
total = response.context['data_stream_total']
progress.append((current, total))
# Act
content = await (await blob.download_blob(raw_response_hook=callback
)).readall()
# Assert
self.assertEqual(byte_data, content)
self.assert_download_progress(len(byte_data),
self.config.max_chunk_get_size,
self.config.max_single_get_size,
progress)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_get_blob_to_stream_with_md5_async(self, resource_group,
location, storage_account,
storage_account_key):
# parallel tests introduce random order of requests, can only run live
# Arrange
await self._setup(storage_account.name, storage_account_key)
blob = self.bsc.get_blob_client(self.container_name, self.byte_blob)
# Act
FILE_PATH = 'lob_to_stream_with_md5_asyncc.temp.dat'
with open(FILE_PATH, 'wb') as stream:
downloader = await blob.download_blob(validate_content=True,
max_concurrency=2)
read_bytes = await downloader.readinto(stream)
# Assert
self.assertEqual(read_bytes, len(self.byte_data))
with open(FILE_PATH, 'rb') as stream:
actual = stream.read()
self.assertEqual(self.byte_data, actual)
self._teardown(FILE_PATH)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_get_blob_with_md5_async(self, resource_group, location,
storage_account,
storage_account_key):
# parallel tests introduce random order of requests, can only run live
# Arrange
await self._setup(storage_account.name, storage_account_key)
blob = self.bsc.get_blob_client(self.container_name, self.byte_blob)
# Act
content = await (await
blob.download_blob(validate_content=True,
max_concurrency=2)).readall()
# Assert
self.assertEqual(self.byte_data, content)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_get_blob_range_to_stream_with_overall_md5_async(
self, resource_group, location, storage_account,
storage_account_key):
# parallel tests introduce random order of requests, can only run live
# Arrange
await self._setup(storage_account.name, storage_account_key)
blob = self.bsc.get_blob_client(self.container_name, self.byte_blob)
props = await blob.get_blob_properties()
props.content_settings.content_md5 = b'MDAwMDAwMDA='
await blob.set_http_headers(props.content_settings)
# Act
FILE_PATH = 'range_to_stream_with_overall_md5_async.temp.dat'
with open(FILE_PATH, 'wb') as stream:
downloader = await blob.download_blob(offset=0,
length=1024,
validate_content=True,
max_concurrency=2)
read_bytes = await downloader.readinto(stream)
# Assert
self.assertEqual(read_bytes, 1024)
self.assertEqual(b'MDAwMDAwMDA=',
downloader.properties.content_settings.content_md5)
self.assertEqual(downloader.size, 1024)
self._teardown(FILE_PATH)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_get_blob_range_with_overall_md5_async(
self, resource_group, location, storage_account,
storage_account_key):
# parallel tests introduce random order of requests, can only run live
await self._setup(storage_account.name, storage_account_key)
blob = self.bsc.get_blob_client(self.container_name, self.byte_blob)
content = await blob.download_blob(offset=0,
length=1024,
validate_content=True)
# Arrange
props = await blob.get_blob_properties()
props.content_settings.content_md5 = b'MDAwMDAwMDA='
await blob.set_http_headers(props.content_settings)
# Act
content = await blob.download_blob(offset=0,
length=1024,
validate_content=True)
# Assert
self.assertEqual(b'MDAwMDAwMDA=',
content.properties.content_settings.content_md5)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_get_blob_range_with_range_md5_async(self, resource_group,
location,
storage_account,
storage_account_key):
# parallel tests introduce random order of requests, can only run live
await self._setup(storage_account.name, storage_account_key)
blob = self.bsc.get_blob_client(self.container_name, self.byte_blob)
content = await blob.download_blob(offset=0,
length=1024,
validate_content=True)
# Arrange
props = await blob.get_blob_properties()
props.content_settings.content_md5 = None
await blob.set_http_headers(props.content_settings)
# Act
content = await blob.download_blob(offset=0,
length=1024,
validate_content=True)
# Assert
self.assertIsNotNone(content.properties.content_settings.content_type)
self.assertIsNone(content.properties.content_settings.content_md5)
self.assertEqual(content.properties.size, 1024)
# ------------------------------------------------------------------------------
class StorageBlobTagsTest(AsyncStorageTestCase):
async def _setup(self, storage_account, key):
self.bsc = BlobServiceClient(self.account_url(storage_account, "blob"), credential=key)
self.container_name = self.get_resource_name("container")
if self.is_live:
container = self.bsc.get_container_client(self.container_name)
try:
await container.create_container(timeout=5)
except ResourceExistsError:
pass
self.byte_data = self.get_random_bytes(1024)
#--Helpers-----------------------------------------------------------------
def _get_blob_reference(self):
return self.get_resource_name(TEST_BLOB_PREFIX)
async def _create_block_blob(self, tags=None, container_name=None, blob_name=None):
blob_name = blob_name or self._get_blob_reference()
blob_client = self.bsc.get_blob_client(container_name or self.container_name, blob_name)
resp = await blob_client.upload_blob(self.byte_data, length=len(self.byte_data), overwrite=True, tags=tags)
return blob_client, resp
async def _create_empty_block_blob(self):
blob_name = self._get_blob_reference()
blob_client = self.bsc.get_blob_client(self.container_name, blob_name)
resp = await blob_client.upload_blob(b'', length=0, overwrite=True)
return blob_client, resp
async def _create_append_blob(self, tags=None):
blob_name = self._get_blob_reference()
blob_client = self.bsc.get_blob_client(self.container_name, blob_name)
resp = await blob_client.create_append_blob(tags=tags)
return blob_client, resp
async def _create_page_blob(self, tags=None):
blob_name = self._get_blob_reference()
blob_client = self.bsc.get_blob_client(self.container_name, blob_name)
resp = await blob_client.create_page_blob(tags=tags, size=512)
return blob_client, resp
async def _create_container(self, prefix="container"):
container_name = self.get_resource_name(prefix)
try:
await self.bsc.create_container(container_name)
except:
pass
return container_name
#-- test cases for blob tags ----------------------------------------------
@GlobalResourceGroupPreparer()
@StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage')
@AsyncStorageTestCase.await_prepared_test
async def test_set_blob_tags(self, resource_group, location, storage_account, storage_account_key):
await self._setup(storage_account, storage_account_key)
blob_client, _ = await self._create_block_blob()
# Act
tags = {"tag1": "firsttag", "tag2": "secondtag", "tag3": "thirdtag"}
resp = await blob_client.set_blob_tags(tags)
# Assert
self.assertIsNotNone(resp)
@pytest.mark.playback_test_only
@GlobalStorageAccountPreparer()
@AsyncStorageTestCase.await_prepared_test
async def test_set_blob_tags_for_a_version(self, resource_group, location, storage_account, storage_account_key):
await self._setup(storage_account, storage_account_key)
# use this version to set tag
blob_client, resp = await self._create_block_blob()
await self._create_block_blob()
# TODO: enable versionid for this account and test set tag for a version
# Act
tags = {"tag1": "firsttag", "tag2": "secondtag", "tag3": "thirdtag"}
resp = await blob_client.set_blob_tags(tags, version_id=resp['version_id'])
# Assert
self.assertIsNotNone(resp)
@GlobalResourceGroupPreparer()
@StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage')
@AsyncStorageTestCase.await_prepared_test
async def test_get_blob_tags(self, resource_group, location, storage_account, storage_account_key):
await self._setup(storage_account, storage_account_key)
blob_client, resp = await self._create_block_blob()
# Act
tags = {"tag1": "firsttag", "tag2": "secondtag", "tag3": "thirdtag"}
await blob_client.set_blob_tags(tags)
resp = await blob_client.get_blob_tags()
# Assert
self.assertIsNotNone(resp)
self.assertEqual(len(resp), 3)
for key, value in resp.items():
self.assertEqual(tags[key], value)
@GlobalResourceGroupPreparer()
@StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage')
@AsyncStorageTestCase.await_prepared_test
async def test_get_blob_tags_for_a_snapshot(self, resource_group, location, storage_account, storage_account_key):
await self._setup(storage_account, storage_account_key)
tags = {"+-./:=_ ": "firsttag", "tag2": "+-./:=_", "+-./:=_1": "+-./:=_"}
blob_client, resp = await self._create_block_blob(tags=tags)
snapshot = await blob_client.create_snapshot()
snapshot_client = self.bsc.get_blob_client(self.container_name, blob_client.blob_name, snapshot=snapshot)
resp = await snapshot_client.get_blob_tags()
# Assert
self.assertIsNotNone(resp)
self.assertEqual(len(resp), 3)
for key, value in resp.items():
self.assertEqual(tags[key], value)
@GlobalResourceGroupPreparer()
@StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage')
@AsyncStorageTestCase.await_prepared_test
async def test_upload_block_blob_with_tags(self, resource_group, location, storage_account, storage_account_key):
await self._setup(storage_account, storage_account_key)
tags = {"tag1": "firsttag", "tag2": "secondtag", "tag3": "thirdtag"}
blob_client, resp = await self._create_block_blob(tags=tags)
resp = await blob_client.get_blob_tags()
# Assert
self.assertIsNotNone(resp)
self.assertEqual(len(resp), 3)
@GlobalResourceGroupPreparer()
@StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage')
@AsyncStorageTestCase.await_prepared_test
async def test_get_blob_properties_returns_tags_num(self, resource_group, location, storage_account, storage_account_key):
await self._setup(storage_account, storage_account_key)
tags = {"tag1": "firsttag", "tag2": "secondtag", "tag3": "thirdtag"}
blob_client, resp = await self._create_block_blob(tags=tags)
resp = await blob_client.get_blob_properties()
downloaded = await blob_client.download_blob()
# Assert
self.assertIsNotNone(resp)
self.assertEqual(resp.tag_count, len(tags))
self.assertEqual(downloaded.properties.tag_count, len(tags))
@GlobalResourceGroupPreparer()
@StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage')
@AsyncStorageTestCase.await_prepared_test
async def test_create_append_blob_with_tags(self, resource_group, location, storage_account, storage_account_key):
await self._setup(storage_account, storage_account_key)
tags = {"+-./:=_ ": "firsttag", "tag2": "+-./:=_", "+-./:=_1": "+-./:=_"}
blob_client, resp = await self._create_append_blob(tags=tags)
resp = await blob_client.get_blob_tags()
# Assert
self.assertIsNotNone(resp)
self.assertEqual(len(resp), 3)
@GlobalResourceGroupPreparer()
@StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage')
@AsyncStorageTestCase.await_prepared_test
async def test_create_page_blob_with_tags(self, resource_group, location, storage_account, storage_account_key):
await self._setup(storage_account, storage_account_key)
tags = {"tag1": "firsttag", "tag2": "secondtag", "tag3": "thirdtag"}
blob_client, resp = await self._create_page_blob(tags=tags)
resp = await blob_client.get_blob_tags()
# Assert
self.assertIsNotNone(resp)
self.assertEqual(len(resp), 3)
@GlobalResourceGroupPreparer()
@StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage')
@AsyncStorageTestCase.await_prepared_test
async def test_commit_block_list_with_tags(self, resource_group, location, storage_account, storage_account_key):
await self._setup(storage_account, storage_account_key)
tags = {"tag1": "firsttag", "tag2": "secondtag", "tag3": "thirdtag"}
blob_client, resp = await self._create_empty_block_blob()
await blob_client.stage_block('1', b'AAA')
await blob_client.stage_block('2', b'BBB')
await blob_client.stage_block('3', b'CCC')
# Act
block_list = [BlobBlock(block_id='1'), BlobBlock(block_id='2'), BlobBlock(block_id='3')]
await blob_client.commit_block_list(block_list, tags=tags)
resp = await blob_client.get_blob_tags()
# Assert
self.assertIsNotNone(resp)
self.assertEqual(len(resp), len(tags))
@GlobalResourceGroupPreparer()
@StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage')
@AsyncStorageTestCase.await_prepared_test
async def test_start_copy_from_url_with_tags(self, resource_group, location, storage_account, storage_account_key):
await self._setup(storage_account, storage_account_key)
tags = {"tag1": "firsttag", "tag2": "secondtag", "tag3": "thirdtag"}
blob_client, resp = await self._create_block_blob()
# Act
sourceblob = '{0}/{1}/{2}'.format(
self.account_url(storage_account, "blob"), self.container_name, blob_client.blob_name)
copyblob = self.bsc.get_blob_client(self.container_name, 'blob1copy')
copy = await copyblob.start_copy_from_url(sourceblob, tags=tags)
# Assert
self.assertIsNotNone(copy)
self.assertEqual(copy['copy_status'], 'success')
self.assertFalse(isinstance(copy['copy_status'], Enum))
self.assertIsNotNone(copy['copy_id'])
copy_content = await (await copyblob.download_blob()).readall()
self.assertEqual(copy_content, self.byte_data)
resp = await copyblob.get_blob_tags()
# Assert
self.assertIsNotNone(resp)
self.assertEqual(len(resp), len(tags))
@GlobalResourceGroupPreparer()
@StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage')
@AsyncStorageTestCase.await_prepared_test
async def test_list_blobs_returns_tags(self, resource_group, location, storage_account, storage_account_key):
await self._setup(storage_account, storage_account_key)
tags = {"tag1": "firsttag", "tag2": "secondtag", "tag3": "thirdtag"}
await self._create_block_blob(tags=tags)
container = self.bsc.get_container_client(self.container_name)
blob_list = container.list_blobs(include="tags")
#Assert
async for blob in blob_list:
self.assertEqual(blob.tag_count, len(tags))
for key, value in blob.tags.items():
self.assertEqual(tags[key], value)
@GlobalResourceGroupPreparer()
@StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage')
@AsyncStorageTestCase.await_prepared_test
async def test_filter_blobs(self, resource_group, location, storage_account, storage_account_key):
await self._setup(storage_account, storage_account_key)
container_name1 = await self._create_container(prefix="container1")
container_name2 = await self._create_container(prefix="container2")
container_name3 = await self._create_container(prefix="container3")
tags = {"tag1": "firsttag", "tag2": "secondtag", "tag3": "thirdtag"}
await self._create_block_blob(tags=tags, blob_name="blob1")
await self._create_block_blob(tags=tags, blob_name="blob2", container_name=container_name1)
await self._create_block_blob(tags=tags, blob_name="blob3", container_name=container_name2)
await self._create_block_blob(tags=tags, blob_name="blob4", container_name=container_name3)
if self.is_live:
sleep(10)
where = "tag1='firsttag'"
blob_list = self.bsc.find_blobs_by_tags(filter_expression=where, results_per_page=2).by_page()
first_page = await blob_list.__anext__()
items_on_page1 = list()
async for item in first_page:
items_on_page1.append(item)
second_page = await blob_list.__anext__()
items_on_page2 = list()
async for item in second_page:
items_on_page2.append(item)
self.assertEqual(2, len(items_on_page1))
self.assertEqual(2, len(items_on_page2))
class StorageLoggingTestAsync(StorageTestCase):
def setUp(self):
super(StorageLoggingTestAsync, self).setUp()
url = self._get_account_url()
credential = self._get_shared_key_credential()
self.bsc = BlobServiceClient(url,
credential=credential,
transport=AiohttpTestTransport())
self.container_name = self.get_resource_name('utcontainer')
def tearDown(self):
if not self.is_playback():
loop = asyncio.get_event_loop()
try:
loop.run_until_complete(
self.bsc.delete_container(self.container_name))
except:
pass
return super(StorageLoggingTestAsync, self).tearDown()
async def _setup(self):
if not self.is_playback():
try:
# create source blob to be copied from
self.source_blob_name = self.get_resource_name('srcblob')
self.source_blob_data = self.get_random_bytes(4 * 1024)
source_blob = self.bsc.get_blob_client(self.container_name,
self.source_blob_name)
await self.bsc.create_container(self.container_name)
await source_blob.upload_blob(self.source_blob_data)
# generate a SAS so that it is accessible with a URL
sas_token = source_blob.generate_shared_access_signature(
permission=BlobPermissions.READ,
expiry=datetime.utcnow() + timedelta(hours=1),
)
sas_source = BlobClient(source_blob.url, credential=sas_token)
self.source_blob_url = sas_source.url
except:
pass
async def _test_authorization_is_scrubbed_off(self):
await self._setup()
# Arrange
container = self.bsc.get_container_client(self.container_name)
# Act
with LogCaptured(self) as log_captured:
await container.get_container_properties(logging_enable=True)
log_as_str = log_captured.getvalue()
# Assert
# make sure authorization header is logged, but its value is not
# the keyword SharedKey is present in the authorization header's value
self.assertTrue(_AUTHORIZATION_HEADER_NAME in log_as_str)
self.assertFalse('SharedKey' in log_as_str)
@record
def test_authorization_is_scrubbed_off(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(self._test_authorization_is_scrubbed_off())
async def _test_sas_signature_is_scrubbed_off(self):
# Test can only run live
if TestMode.need_recording_file(self.test_mode):
return
await self._setup()
# Arrange
container = self.bsc.get_container_client(self.container_name)
token = container.generate_shared_access_signature(
permission=ContainerPermissions.READ,
expiry=datetime.utcnow() + timedelta(hours=1),
)
# parse out the signed signature
token_components = parse_qs(token)
signed_signature = quote(
token_components[QueryStringConstants.SIGNED_SIGNATURE][0])
sas_service = ContainerClient(container.url, credential=token)
# Act
with LogCaptured(self) as log_captured:
await sas_service.get_account_information(logging_enable=True)
log_as_str = log_captured.getvalue()
# Assert
# make sure the query parameter 'sig' is logged, but its value is not
self.assertTrue(
QueryStringConstants.SIGNED_SIGNATURE in log_as_str)
self.assertFalse(signed_signature in log_as_str)
@record
def test_sas_signature_is_scrubbed_off(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(self._test_sas_signature_is_scrubbed_off())
async def _test_copy_source_sas_is_scrubbed_off(self):
# Test can only run live
if TestMode.need_recording_file(self.test_mode):
return
await self._setup()
# Arrange
dest_blob_name = self.get_resource_name('destblob')
dest_blob = self.bsc.get_blob_client(self.container_name,
dest_blob_name)
# parse out the signed signature
token_components = parse_qs(self.source_blob_url)
signed_signature = quote(
token_components[QueryStringConstants.SIGNED_SIGNATURE][0])
# Act
with LogCaptured(self) as log_captured:
await dest_blob.start_copy_from_url(self.source_blob_url,
requires_sync=True,
logging_enable=True)
log_as_str = log_captured.getvalue()
# Assert
# make sure the query parameter 'sig' is logged, but its value is not
self.assertTrue(
QueryStringConstants.SIGNED_SIGNATURE in log_as_str)
self.assertFalse(signed_signature in log_as_str)
# make sure authorization header is logged, but its value is not
# the keyword SharedKey is present in the authorization header's value
self.assertTrue(_AUTHORIZATION_HEADER_NAME in log_as_str)
self.assertFalse('SharedKey' in log_as_str)
@record
def test_copy_source_sas_is_scrubbed_off(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(self._test_copy_source_sas_is_scrubbed_off())
class BlobStorageAccountTestAsync(StorageTestCase):
def setUp(self):
super(BlobStorageAccountTestAsync, self).setUp()
url = self._get_account_url()
credential = self._get_shared_key_credential()
self.bsc = BlobServiceClient(url, credential=credential, transport=AiohttpTestTransport())
self.container_name = self.get_resource_name('utcontainer')
# if not self.is_playback():
# self.bsc.create_container(self.container_name)
def tearDown(self):
if not self.is_playback():
loop = asyncio.get_event_loop()
try:
loop.run_until_complete(self.bsc.delete_container(self.container_name))
except:
pass
return super(BlobStorageAccountTestAsync, self).tearDown()
# --Helpers-----------------------------------------------------------------
async def _setup(self):
if not self.is_playback():
try:
await self.bsc.create_container(self.container_name)
except:
pass
def _get_blob_reference(self):
blob_name = self.get_resource_name(TEST_BLOB_PREFIX)
return self.bsc.get_blob_client(self.container_name, blob_name)
async def _create_blob(self):
blob = self._get_blob_reference()
await blob.upload_blob(b'')
return blob
async def assertBlobEqual(self, container_name, blob_name, expected_data):
blob = self.bsc.get_blob_client(container_name, blob_name)
actual_data = await blob.download_blob().content_as_bytes()
self.assertEqual(actual_data, expected_data)
# --Tests specific to Blob Storage Accounts (not general purpose)------------
async def _test_standard_blob_tier_set_tier_api(self):
await self._setup()
container = self.bsc.get_container_client(self.container_name)
tiers = [StandardBlobTier.Archive, StandardBlobTier.Cool, StandardBlobTier.Hot]
for tier in tiers:
blob = self._get_blob_reference()
data = b'hello world'
await blob.upload_blob(data)
blob_ref = await blob.get_blob_properties()
self.assertIsNotNone(blob_ref.blob_tier)
self.assertTrue(blob_ref.blob_tier_inferred)
self.assertIsNone(blob_ref.blob_tier_change_time)
blobs = []
async for b in container.list_blobs():
blobs.append(b)
# Assert
self.assertIsNotNone(blobs)
self.assertGreaterEqual(len(blobs), 1)
self.assertIsNotNone(blobs[0])
self.assertNamedItemInContainer(blobs, blob.blob_name)
self.assertIsNotNone(blobs[0].blob_tier)
self.assertTrue(blobs[0].blob_tier_inferred)
self.assertIsNone(blobs[0].blob_tier_change_time)
await blob.set_standard_blob_tier(tier)
blob_ref2 = await blob.get_blob_properties()
self.assertEqual(tier, blob_ref2.blob_tier)
self.assertFalse(blob_ref2.blob_tier_inferred)
self.assertIsNotNone(blob_ref2.blob_tier_change_time)
blobs = []
async for b in container.list_blobs():
blobs.append(b)
# Assert
self.assertIsNotNone(blobs)
self.assertGreaterEqual(len(blobs), 1)
self.assertIsNotNone(blobs[0])
self.assertNamedItemInContainer(blobs, blob.blob_name)
self.assertEqual(blobs[0].blob_tier, tier)
self.assertFalse(blobs[0].blob_tier_inferred)
self.assertIsNotNone(blobs[0].blob_tier_change_time)
await blob.delete_blob()
@record
def test_standard_blob_tier_set_tier_api(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(self._test_standard_blob_tier_set_tier_api())
async def _test_rehydration_status(self):
await self._setup()
blob_name = 'rehydration_test_blob_1'
blob_name2 = 'rehydration_test_blob_2'
container = self.bsc.get_container_client(self.container_name)
data = b'hello world'
blob = await container.upload_blob(blob_name, data)
await blob.set_standard_blob_tier(StandardBlobTier.Archive)
await blob.set_standard_blob_tier(StandardBlobTier.Cool)
blob_ref = await blob.get_blob_properties()
self.assertEqual(StandardBlobTier.Archive, blob_ref.blob_tier)
self.assertEqual("rehydrate-pending-to-cool", blob_ref.archive_status)
self.assertFalse(blob_ref.blob_tier_inferred)
blobs = []
async for b in container.list_blobs():
blobs.append(b)
await blob.delete_blob()
# Assert
self.assertIsNotNone(blobs)
self.assertGreaterEqual(len(blobs), 1)
self.assertIsNotNone(blobs[0])
self.assertNamedItemInContainer(blobs, blob.blob_name)
self.assertEqual(StandardBlobTier.Archive, blobs[0].blob_tier)
self.assertEqual("rehydrate-pending-to-cool", blobs[0].archive_status)
self.assertFalse(blobs[0].blob_tier_inferred)
blob2 = await container.upload_blob(blob_name2, data)
await blob2.set_standard_blob_tier(StandardBlobTier.Archive)
await blob2.set_standard_blob_tier(StandardBlobTier.Hot)
blob_ref2 = await blob2.get_blob_properties()
self.assertEqual(StandardBlobTier.Archive, blob_ref2.blob_tier)
self.assertEqual("rehydrate-pending-to-hot", blob_ref2.archive_status)
self.assertFalse(blob_ref2.blob_tier_inferred)
blobs = []
async for b in container.list_blobs():
blobs.append(b)
# Assert
self.assertIsNotNone(blobs)
self.assertGreaterEqual(len(blobs), 1)
self.assertIsNotNone(blobs[0])
self.assertNamedItemInContainer(blobs, blob2.blob_name)
self.assertEqual(StandardBlobTier.Archive, blobs[0].blob_tier)
self.assertEqual("rehydrate-pending-to-hot", blobs[0].archive_status)
self.assertFalse(blobs[0].blob_tier_inferred)
@record
def test_rehydration_status(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(self._test_rehydration_status())
