async def test_commit_block_list_with_tags(self, resource_group, location,
storage_account,
storage_account_key):
await self._setup(storage_account, storage_account_key)
tags = {"tag1": "firsttag", "tag2": "secondtag", "tag3": "thirdtag"}
blob_client, resp = await self._create_empty_block_blob(
tags={'condition tag': 'test tag'})
await blob_client.stage_block('1', b'AAA')
await blob_client.stage_block('2', b'BBB')
await blob_client.stage_block('3', b'CCC')
# Act
block_list = [
BlobBlock(block_id='1'),
BlobBlock(block_id='2'),
BlobBlock(block_id='3')
]
with self.assertRaises(ResourceModifiedError):
await blob_client.commit_block_list(
block_list,
tags=tags,
if_tags_match_condition="\"condition tag\"='wrong tag'")
await blob_client.commit_block_list(
block_list,
tags=tags,
if_tags_match_condition="\"condition tag\"='test tag'")
resp = await blob_client.get_blob_tags()
# Assert
self.assertIsNotNone(resp)
self.assertEqual(len(resp), len(tags))
def test_put_block_and_put_block_list(self):
# Arrange
blob_client, _ = self._create_block_blob()
blob_client.stage_block('1', b'AAA', cpk=TEST_ENCRYPTION_KEY)
blob_client.stage_block('2', b'BBB', cpk=TEST_ENCRYPTION_KEY)
blob_client.stage_block('3', b'CCC', cpk=TEST_ENCRYPTION_KEY)
# Act
block_list = [BlobBlock(block_id='1'), BlobBlock(block_id='2'), BlobBlock(block_id='3')]
put_block_list_resp = blob_client.commit_block_list(block_list,
cpk=TEST_ENCRYPTION_KEY)
# Assert
self.assertIsNotNone(put_block_list_resp['etag'])
self.assertIsNotNone(put_block_list_resp['last_modified'])
self.assertTrue(put_block_list_resp['request_server_encrypted'])
self.assertEqual(put_block_list_resp['encryption_key_sha256'], TEST_ENCRYPTION_KEY.key_hash)
# Act get the blob content without cpk should fail
with self.assertRaises(HttpResponseError):
blob_client.download_blob()
# Act get the blob content
blob = blob_client.download_blob(cpk=TEST_ENCRYPTION_KEY)
# Assert content was retrieved with the cpk
self.assertEqual(blob.content_as_bytes(), b'AAABBBCCC')
self.assertEqual(blob.properties.etag, put_block_list_resp['etag'])
self.assertEqual(blob.properties.last_modified, put_block_list_resp['last_modified'])
self.assertEqual(blob.properties.encryption_key_sha256, TEST_ENCRYPTION_KEY.key_hash)
def test_put_block_from_url_and_commit(self):
# Arrange
dest_blob_name = self.get_resource_name('destblob')
# Act part 1: make put block from url calls
self.bs.put_block_from_url(self.container_name,
dest_blob_name,
self.source_blob_url,
source_range_start=0,
source_range_end=4 * 1024 - 1,
block_id=1)
self.bs.put_block_from_url(self.container_name,
dest_blob_name,
self.source_blob_url,
source_range_start=4 * 1024,
source_range_end=8 * 1024,
block_id=2)
# Assert blocks
block_list = self.bs.get_block_list(self.container_name,
dest_blob_name, None, 'all')
self.assertEqual(len(block_list.uncommitted_blocks), 2)
self.assertEqual(len(block_list.committed_blocks), 0)
# Act part 2: commit the blocks
block_list = [BlobBlock(id='1'), BlobBlock(id='2')]
self.bs.put_block_list(self.container_name, dest_blob_name, block_list)
# Assert destination blob has right content
blob = self.bs.get_blob_to_bytes(self.container_name, dest_blob_name)
self.assertEqual(blob.content, self.source_blob_data)
def test_get_block_list_committed_blocks(self):
# Arrange
blob_name = self._get_blob_reference()
self.bs.put_block(self.container_name, blob_name, b'AAA', '1')
self.bs.put_block(self.container_name, blob_name, b'BBB', '2')
self.bs.put_block(self.container_name, blob_name, b'CCC', '3')
block_list = [BlobBlock(id='1'), BlobBlock(id='2'), BlobBlock(id='3')]
self.bs.put_block_list(self.container_name, blob_name, block_list)
# Act
block_list = self.bs.get_block_list(self.container_name, blob_name,
None, 'all')
# Assert
self.assertIsNotNone(block_list)
self.assertIsInstance(block_list, BlobBlockList)
self.assertEqual(len(block_list.uncommitted_blocks), 0)
self.assertEqual(len(block_list.committed_blocks), 3)
self.assertEqual(block_list.committed_blocks[0].id, '1')
self.assertEqual(block_list.committed_blocks[0].size, 3)
self.assertEqual(block_list.committed_blocks[1].id, '2')
self.assertEqual(block_list.committed_blocks[1].size, 3)
self.assertEqual(block_list.committed_blocks[2].id, '3')
self.assertEqual(block_list.committed_blocks[2].size, 3)
def test_get_block_list_committed_blocks(self, resource_group, location, storage_account, storage_account_key):
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
blob.stage_block('1', b'AAA')
blob.stage_block('2', b'BBB')
blob.stage_block('3', b'CCC')
block_list = [BlobBlock(block_id='1'), BlobBlock(block_id='2'), BlobBlock(block_id='3')]
blob.commit_block_list(block_list)
# Act
block_list = blob.get_block_list('committed')
# Assert
self.assertIsNotNone(block_list)
self.assertEqual(len(block_list), 2)
self.assertEqual(len(block_list[1]), 0)
self.assertEqual(len(block_list[0]), 3)
self.assertEqual(block_list[0][0].id, '1')
self.assertEqual(block_list[0][0].size, 3)
self.assertEqual(block_list[0][1].id, '2')
self.assertEqual(block_list[0][1].size, 3)
self.assertEqual(block_list[0][2].id, '3')
self.assertEqual(block_list[0][2].size, 3)
def test_get_block_list_committed_blocks(self):
# Arrange
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
blob.stage_block('1', b'AAA')
blob.stage_block('2', b'BBB')
blob.stage_block('3', b'CCC')
block_list = [
BlobBlock(block_id='1'),
BlobBlock(block_id='2'),
BlobBlock(block_id='3')
]
blob.commit_block_list(block_list)
# Act
block_list = blob.get_block_list('committed')
# Assert
self.assertIsNotNone(block_list)
self.assertEqual(len(block_list), 2)
self.assertEqual(len(block_list[1]), 0)
self.assertEqual(len(block_list[0]), 3)
self.assertEqual(block_list[0][0].id, '1')
self.assertEqual(block_list[0][0].size, 3)
self.assertEqual(block_list[0][1].id, '2')
self.assertEqual(block_list[0][1].size, 3)
self.assertEqual(block_list[0][2].id, '3')
self.assertEqual(block_list[0][2].size, 3)
async def _test_put_block_list(self):
# Arrange
await self._setup()
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
await blob.stage_block('1', b'AAA')
await blob.stage_block('2', b'BBB')
await blob.stage_block('3', b'CCC')
# Act
block_list = [
BlobBlock(block_id='1'),
BlobBlock(block_id='2'),
BlobBlock(block_id='3')
]
put_block_list_resp = await blob.commit_block_list(block_list)
# Assert
content = await blob.download_blob()
actual = await content.readall()
self.assertEqual(actual, b'AAABBBCCC')
self.assertEqual(content.properties.etag,
put_block_list_resp.get('etag'))
self.assertEqual(content.properties.last_modified,
put_block_list_resp.get('last_modified'))
async def _test_put_block_from_url_and_commit(self):
# Arrange
# create source blob and get source blob url
source_blob_name = self.get_resource_name("sourceblob")
self.config.use_byte_buffer = True # Make sure using chunk upload, then we can record the request
source_blob_client, _ = await self._create_block_blob(blob_name=source_blob_name, data=self.byte_data)
source_blob_sas = generate_blob_sas(
source_blob_client.account_name,
source_blob_client.container_name,
source_blob_client.blob_name,
snapshot=source_blob_client.snapshot,
account_key=source_blob_client.credential.account_key,
permission=BlobSasPermissions(read=True),
expiry=datetime.utcnow() + timedelta(hours=1)
)
source_blob_url = source_blob_client.url + "?" + source_blob_sas
# create destination blob
self.config.use_byte_buffer = False
destination_blob_client, _ = await self._create_block_blob(cpk=TEST_ENCRYPTION_KEY)
# Act part 1: make put block from url calls
await destination_blob_client.stage_block_from_url(block_id=1, source_url=source_blob_url,
source_offset=0, source_length=4 * 1024,
cpk=TEST_ENCRYPTION_KEY)
await destination_blob_client.stage_block_from_url(block_id=2, source_url=source_blob_url,
source_offset=4 * 1024, source_length=4 * 1024,
cpk=TEST_ENCRYPTION_KEY)
# Assert blocks
committed, uncommitted = await destination_blob_client.get_block_list('all')
self.assertEqual(len(uncommitted), 2)
self.assertEqual(len(committed), 0)
# commit the blocks without cpk should fail
block_list = [BlobBlock(block_id='1'), BlobBlock(block_id='2')]
with self.assertRaises(HttpResponseError):
await destination_blob_client.commit_block_list(block_list)
# Act commit the blocks with cpk should succeed
put_block_list_resp = await destination_blob_client.commit_block_list(block_list,
cpk=TEST_ENCRYPTION_KEY)
# Assert
self.assertIsNotNone(put_block_list_resp['etag'])
self.assertIsNotNone(put_block_list_resp['last_modified'])
self.assertTrue(put_block_list_resp['request_server_encrypted'])
self.assertEqual(put_block_list_resp['encryption_key_sha256'], TEST_ENCRYPTION_KEY.key_hash)
# Act get the blob content
blob = await destination_blob_client.download_blob(cpk=TEST_ENCRYPTION_KEY)
# Assert content was retrieved with the cpk
self.assertEqual(await blob.readall(), self.byte_data[0: 8 * 1024])
self.assertEqual(blob.properties.etag, put_block_list_resp['etag'])
self.assertEqual(blob.properties.last_modified, put_block_list_resp['last_modified'])
self.assertEqual(blob.properties.encryption_key_sha256, TEST_ENCRYPTION_KEY.key_hash)
def test_put_block_from_url_and_commit(self):
# Arrange
dest_blob_name = self._get_blob_reference()
# Act part 1: make put block from url calls
self.bbs.put_block_from_url(self.container_name,
dest_blob_name,
self.source_blob_url,
source_range_start=0,
source_range_end=4 * 1024 - 1,
block_id=1,
cpk=TEST_ENCRYPTION_KEY)
self.bbs.put_block_from_url(self.container_name,
dest_blob_name,
self.source_blob_url,
source_range_start=4 * 1024,
source_range_end=8 * 1024,
block_id=2,
cpk=TEST_ENCRYPTION_KEY)
# Assert blocks
block_list = self.bbs.get_block_list(self.container_name,
dest_blob_name, None, 'all')
self.assertEqual(len(block_list.uncommitted_blocks), 2)
self.assertEqual(len(block_list.committed_blocks), 0)
# commit the blocks without cpk should fail
block_list = [BlobBlock(id='1'), BlobBlock(id='2')]
with self.assertRaises(AzureHttpError):
self.bbs.put_block_list(self.container_name, dest_blob_name,
block_list)
# Act commit the blocks with cpk should succeed
put_block_list_resp = self.bbs.put_block_list(self.container_name,
dest_blob_name,
block_list,
cpk=TEST_ENCRYPTION_KEY)
# Assert
self.assertIsNotNone(put_block_list_resp.etag)
self.assertIsNotNone(put_block_list_resp.last_modified)
self.assertTrue(put_block_list_resp.request_server_encrypted)
self.assertEqual(put_block_list_resp.encryption_key_sha256,
TEST_ENCRYPTION_KEY.key_hash)
# Assert destination blob has right content
blob = self.bbs.get_blob_to_bytes(self.container_name,
dest_blob_name,
cpk=TEST_ENCRYPTION_KEY)
self.assertEqual(blob.content, self.byte_data[0:8 * 1024 + 1])
self.assertEqual(blob.properties.etag, put_block_list_resp.etag)
self.assertEqual(blob.properties.last_modified,
put_block_list_resp.last_modified)
self.assertEqual(blob.properties.encryption_key_sha256,
TEST_ENCRYPTION_KEY.key_hash)
def test_put_block_list_with_md5(self):
# Arrange
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
blob.stage_block('1', b'AAA')
blob.stage_block('2', b'BBB')
blob.stage_block('3', b'CCC')
# Act
block_list = [BlobBlock(block_id='1'), BlobBlock(block_id='2'), BlobBlock(block_id='3')]
blob.commit_block_list(block_list, validate_content=True)
def test_put_block_list_with_md5(self, resource_group, location, storage_account, storage_account_key):
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
blob.stage_block('1', b'AAA')
blob.stage_block('2', b'BBB')
blob.stage_block('3', b'CCC')
# Act
block_list = [BlobBlock(block_id='1'), BlobBlock(block_id='2'), BlobBlock(block_id='3')]
blob.commit_block_list(block_list, validate_content=True)
def test_put_block_list_with_md5(self):
# Arrange
blob_name = self._get_blob_reference()
self.bs.put_block(self.container_name, blob_name, b'AAA', '1')
self.bs.put_block(self.container_name, blob_name, b'BBB', '2')
self.bs.put_block(self.container_name, blob_name, b'CCC', '3')
# Act
block_list = [BlobBlock(id='1'), BlobBlock(id='2'), BlobBlock(id='3')]
self.bs.put_block_list(self.container_name,
blob_name,
block_list,
validate_content=True)
async def test_put_block_and_put_block_list(self, resource_group, location,
storage_account,
storage_account_key):
# Arrange
# test chunking functionality by reducing the size of each chunk,
# otherwise the tests would take too long to execute
bsc = BlobServiceClient(
self.account_url(storage_account.name, "blob"),
storage_account_key,
max_single_put_size=1024,
min_large_block_upload_threshold=1024,
max_block_size=1024,
max_page_size=1024,
transport=AiohttpTestTransport(connection_data_block_size=1024))
await self._setup(bsc)
self.container_name = self.get_resource_name('utcontainer')
blob_client, _ = await self._create_block_blob(bsc)
await blob_client.stage_block('1', b'AAA', cpk=TEST_ENCRYPTION_KEY)
await blob_client.stage_block('2', b'BBB', cpk=TEST_ENCRYPTION_KEY)
await blob_client.stage_block('3', b'CCC', cpk=TEST_ENCRYPTION_KEY)
# Act
block_list = [
BlobBlock(block_id='1'),
BlobBlock(block_id='2'),
BlobBlock(block_id='3')
]
put_block_list_resp = await blob_client.commit_block_list(
block_list, cpk=TEST_ENCRYPTION_KEY)
# Assert
self.assertIsNotNone(put_block_list_resp['etag'])
self.assertIsNotNone(put_block_list_resp['last_modified'])
self.assertTrue(put_block_list_resp['request_server_encrypted'])
self.assertEqual(put_block_list_resp['encryption_key_sha256'],
TEST_ENCRYPTION_KEY.key_hash)
# Act get the blob content without cpk should fail
with self.assertRaises(HttpResponseError):
await blob_client.download_blob()
# Act get the blob content
blob = await blob_client.download_blob(cpk=TEST_ENCRYPTION_KEY)
# Assert content was retrieved with the cpk
self.assertEqual(await blob.readall(), b'AAABBBCCC')
self.assertEqual(blob.properties.etag, put_block_list_resp['etag'])
self.assertEqual(blob.properties.last_modified,
put_block_list_resp['last_modified'])
self.assertEqual(blob.properties.encryption_key_sha256,
TEST_ENCRYPTION_KEY.key_hash)
def test_put_block_and_put_block_list(self, resource_group, location,
storage_account,
storage_account_key):
# Arrange
bsc = BlobServiceClient(self.account_url(storage_account, "blob"),
credential=storage_account_key,
connection_data_block_size=1024,
max_single_put_size=1024,
min_large_block_upload_threshold=1024,
max_block_size=1024,
max_page_size=1024)
self._setup(bsc)
blob_client, _ = self._create_block_blob(bsc)
blob_client.stage_block('1',
b'AAA',
encryption_scope=TEST_ENCRYPTION_KEY_SCOPE)
blob_client.stage_block('2',
b'BBB',
encryption_scope=TEST_ENCRYPTION_KEY_SCOPE)
blob_client.stage_block('3',
b'CCC',
encryption_scope=TEST_ENCRYPTION_KEY_SCOPE)
# Act
block_list = [
BlobBlock(block_id='1'),
BlobBlock(block_id='2'),
BlobBlock(block_id='3')
]
put_block_list_resp = blob_client.commit_block_list(
block_list, encryption_scope=TEST_ENCRYPTION_KEY_SCOPE)
# Assert
self.assertIsNotNone(put_block_list_resp['etag'])
self.assertIsNotNone(put_block_list_resp['last_modified'])
self.assertTrue(put_block_list_resp['request_server_encrypted'])
self.assertEqual(put_block_list_resp['encryption_scope'],
TEST_ENCRYPTION_KEY_SCOPE)
# Act get the blob content
blob = blob_client.download_blob()
# Assert content was retrieved with the cpk
self.assertEqual(blob.readall(), b'AAABBBCCC')
self.assertEqual(blob.properties.etag, put_block_list_resp['etag'])
self.assertEqual(blob.properties.last_modified,
put_block_list_resp['last_modified'])
self.assertEqual(blob.properties.encryption_scope,
TEST_ENCRYPTION_KEY_SCOPE)
self._teardown(bsc)
def test_put_block_list_invalid_block_id(self):
# Arrange
blob_name = self._get_blob_reference()
self.bs.put_block(self.container_name, blob_name, b'AAA', '1')
self.bs.put_block(self.container_name, blob_name, b'BBB', '2')
self.bs.put_block(self.container_name, blob_name, b'CCC', '3')
# Act
try:
block_list = [ BlobBlock(id='1'), BlobBlock(id='2'), BlobBlock(id='4')]
self.bs.put_block_list(self.container_name, blob_name, block_list)
self.fail()
except AzureHttpError as e:
self.assertGreaterEqual(str(e).find('specified block list is invalid'), 0)
def blocks(self):
container_name = self._create_container()
blob_name = self._get_blob_reference()
# Put block
# Block id's must be the same length
self.service.put_block(container_name, blob_name, b'AAA', '1')
self.service.put_block(container_name, blob_name, b'BBB', '2')
self.service.put_block(container_name, blob_name, b'CCC', '3')
# Get Block List
# Defaults to committed only, specify all to get committed and uncommitted
block_list = self.service.get_block_list(
container_name, blob_name, block_list_type=BlockListType.All)
uncommitted = len(block_list.uncommitted_blocks) # 3
committed = len(block_list.committed_blocks) # 0
# Note the blob does not yet appears as blocks have not been committed
exists = self.service.exists(container_name, blob_name) # False
# Commit the blocks
# BlockBlock state defaults to Latest meaning the uncommitted and then
# the committed list is searched for the block id to commit
block_list = [BlobBlock(id='1'), BlobBlock(id='2'), BlobBlock(id='3')]
self.service.put_block_list(container_name, blob_name, block_list)
# Get Block List
# Defaults to committed only, specify all to get committed and uncommitted
block_list = self.service.get_block_list(
container_name, blob_name, block_list_type=BlockListType.All)
uncommitted = len(block_list.uncommitted_blocks) # 0
committed = len(block_list.committed_blocks) # 3
# Add a block
# Put the block
self.service.put_block(container_name, blob_name, b'DDD', '4')
# Get the existing blocks
block_list = self.service.get_block_list(
container_name, blob_name, block_list_type=BlockListType.All)
uncommitted = len(block_list.uncommitted_blocks) # 1
committed = len(block_list.committed_blocks) # 3
# Added the new block to the existing list and commit
new_block_list = block_list.committed_blocks
new_block_list.append(block_list.uncommitted_blocks[0])
self.service.put_block_list(container_name, blob_name, new_block_list)
self.service.delete_container(container_name)
def test_put_block_list(self):
# Arrange
blob_name = self._get_blob_reference()
self.bs.put_block(self.container_name, blob_name, b'AAA', '1')
self.bs.put_block(self.container_name, blob_name, b'BBB', '2')
self.bs.put_block(self.container_name, blob_name, b'CCC', '3')
# Act
block_list = [BlobBlock(id='1'), BlobBlock(id='2'), BlobBlock(id='3')]
self.bs.put_block_list(self.container_name, blob_name, block_list)
# Assert
blob = self.bs.get_blob_to_bytes(self.container_name, blob_name)
self.assertEqual(blob.content, b'AAABBBCCC')
def test_put_block_and_put_block_list(self):
# Arrange
blob_name = self._get_blob_reference()
self.bbs.put_block(self.container_name,
blob_name,
b'AAA',
'1',
cpk=TEST_ENCRYPTION_KEY)
self.bbs.put_block(self.container_name,
blob_name,
b'BBB',
'2',
cpk=TEST_ENCRYPTION_KEY)
self.bbs.put_block(self.container_name,
blob_name,
b'CCC',
'3',
cpk=TEST_ENCRYPTION_KEY)
# Act
block_list = [BlobBlock(id='1'), BlobBlock(id='2'), BlobBlock(id='3')]
put_block_list_resp = self.bbs.put_block_list(self.container_name,
blob_name,
block_list,
cpk=TEST_ENCRYPTION_KEY)
# Assert
self.assertIsNotNone(put_block_list_resp.etag)
self.assertIsNotNone(put_block_list_resp.last_modified)
self.assertTrue(put_block_list_resp.request_server_encrypted)
self.assertEqual(put_block_list_resp.encryption_key_sha256,
TEST_ENCRYPTION_KEY.key_hash)
# Act get the blob content without cpk should fail
with self.assertRaises(AzureHttpError):
self.bbs.get_blob_to_bytes(self.container_name, blob_name)
# Act get the blob content
blob = self.bbs.get_blob_to_bytes(self.container_name,
blob_name,
cpk=TEST_ENCRYPTION_KEY)
# Assert content was retrieved with the cpk
self.assertEqual(blob.content, b'AAABBBCCC')
self.assertEqual(blob.properties.etag, put_block_list_resp.etag)
self.assertEqual(blob.properties.last_modified,
put_block_list_resp.last_modified)
self.assertEqual(blob.properties.encryption_key_sha256,
TEST_ENCRYPTION_KEY.key_hash)
def test_put_block_list_invalid_block_id(self, resource_group, location, storage_account, storage_account_key):
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
blob.stage_block('1', b'AAA')
blob.stage_block('2', b'BBB')
blob.stage_block('3', b'CCC')
# Act
try:
block_list = [BlobBlock(block_id='1'), BlobBlock(block_id='2'), BlobBlock(block_id='4')]
blob.commit_block_list(block_list)
self.fail()
except HttpResponseError as e:
self.assertGreaterEqual(str(e).find('specified block list is invalid'), 0)
async def test_put_block_stream_largest(self, storage_account_name,
storage_account_key):
await self._setup(storage_account_name, storage_account_key)
blob = await self._create_blob()
# Act
stream = LargeStream(LARGEST_BLOCK_SIZE)
blockId = str(uuid.uuid4())
requestId = str(uuid.uuid4())
resp = await blob.stage_block(blockId,
stream,
length=LARGEST_BLOCK_SIZE,
client_request_id=requestId)
await blob.commit_block_list([BlobBlock(blockId)])
block_list = await blob.get_block_list()
# Assert
self.assertIsNotNone(resp)
assert 'content_md5' in resp
assert 'content_crc64' in resp
assert 'request_id' in resp
self.assertIsNotNone(block_list)
self.assertEqual(len(block_list), 2)
self.assertEqual(len(block_list[1]), 0)
self.assertEqual(len(block_list[0]), 1)
self.assertEqual(block_list[0][0].size, LARGEST_BLOCK_SIZE)
def __copy(args):
s3cli = boto3.resource("s3",
aws_access_key_id=args.source_secret_id,
aws_secret_access_key=args.source_secret_key)
azblob = BlockBlobService(args.destination_account, args.destination_SAS)
s3object = s3cli.Object(args.source_bucket, args.source_file)
print("Opening S3 object {0}/{1}".format(args.source_bucket,
args.source_file))
chunk = s3object.get(PartNumber=1)
nchunks = chunk['PartsCount']
blocks = []
for x in range(1, nchunks + 1):
chunk = s3object.get(PartNumber=x)
print("Reading part {0}/{1}".format(x, nchunks))
part = chunk['Body'].read()
print("Writing part {0}/{1}. Size: {2} bytes".format(
x, nchunks, len(part)))
blockid = str(uuid.uuid4())
azblob.put_block(args.destination_container, args.destination_file,
part, blockid)
blocks.append(BlobBlock(id=blockid))
print("Committing file {0}/{1}".format(args.destination_container,
args.destination_file))
azblob.put_block_list(args.destination_container, args.destination_file,
blocks)
print("Committed")
async def test_put_block_bytes_largest_without_network(self, resource_group, location, storage_account, storage_account_key):
payload_dropping_policy = PayloadDroppingPolicy()
credential_policy = _format_shared_key_credential(storage_account.name, storage_account_key)
await self._setup(storage_account, storage_account_key, [payload_dropping_policy, credential_policy])
blob = await self._create_blob()
# Act
data = urandom(LARGEST_BLOCK_SIZE)
blockId = str(uuid.uuid4()).encode('utf-8')
resp = await blob.stage_block(
blockId,
data,
length=LARGEST_BLOCK_SIZE)
await blob.commit_block_list([BlobBlock(blockId)])
block_list = await blob.get_block_list()
# Assert
self.assertIsNotNone(resp)
assert 'content_md5' in resp
assert 'content_crc64' in resp
assert 'request_id' in resp
self.assertIsNotNone(block_list)
self.assertEqual(len(block_list), 2)
self.assertEqual(len(block_list[1]), 0)
self.assertEqual(len(block_list[0]), 1)
self.assertEqual(payload_dropping_policy.put_block_counter, 1)
self.assertEqual(payload_dropping_policy.put_block_sizes[0], LARGEST_BLOCK_SIZE)
def parallel_chunky_upload(self,
container_name,
full_blob_name,
data,
chunks=5):
debug = False
threads = []
block_ids = []
chunk_size = len(data) / chunks
chunks = [
data[i:i + chunk_size] for i in xrange(0, len(data), chunk_size)
]
for chunk in chunks:
uid = self.generate_uid()
block_ids.append(BlobBlock(id=uid))
t = threading.Thread(target=self._upload_block,
args=(
container_name,
full_blob_name,
chunk,
uid,
))
threads.append(t)
t.start()
[t.join() for t in threads]
self.service.put_block_list(container_name, full_blob_name, block_ids)
return full_blob_name
def test_retry_put_block_with_non_seekable_stream(self):
if TestMode.need_recording_file(self.test_mode):
return
# Arrange
blob_name = self.get_resource_name('blob')
data = self.get_random_bytes(PUT_BLOCK_SIZE)
data_stream = self.NonSeekableStream(BytesIO(data))
# rig the response so that it fails for a single time
self.bs.response_callback = ResponseCallback(status=201, new_status=408).override_first_status
self.bs.retry = ExponentialRetry(initial_backoff=1, increment_base=2, max_attempts=3).retry
# Act
# Note: put_block transforms non-seekable streams into byte arrays before handing it off to the executor
self.bs.put_block(self.container_name, blob_name, data_stream, 1)
# Assert
block_list = self.bs.get_block_list(self.container_name, blob_name, block_list_type="uncommitted")
self.assertEqual(len(block_list.uncommitted_blocks), 1)
self.assertEqual(block_list.uncommitted_blocks[0].size, PUT_BLOCK_SIZE)
# Commit block and verify content
block_list = [BlobBlock(id='1')]
self.bs.put_block_list(self.container_name, blob_name, block_list)
# Assert
blob = self.bs.get_blob_to_bytes(self.container_name, blob_name)
self.assertEqual(blob.content, data)
def save_to_blob(local_path: str, remote_name: str,
container_name: str) -> None:
# Instantiate a new BlobServiceClient using a connection string
blob_service_client = BlobServiceClient.from_connection_string(
connection_string)
# Instantiate a new ContainerClient
container_client = blob_service_client.get_container_client(container_name)
# Ensure container is created
# container_client.create_container()
# Instantiate a new BlobClient
blob_client = container_client.get_blob_client(remote_name)
# Upload content to block blob
# with open(local_path, "rb") as data:
# blob_client.upload_blob(data, timeout=600)
# TODO do something like this instead
block_list = []
chunk_size = 4194304
with open(local_path, "rb") as data:
while True:
read_data = data.read(chunk_size)
if not read_data:
break
blk_id = uuid_to_str(uuid.uuid4())
blob_client.stage_block(block_id=blk_id, data=read_data)
block_list.append(BlobBlock(block_id=blk_id))
blob_client.commit_block_list(block_list)
async def test_put_block_stream_largest_without_network(
self, storage_account_name, storage_account_key):
payload_dropping_policy = PayloadDroppingPolicy()
credential_policy = _format_shared_key_credential(
storage_account_name, storage_account_key)
await self._setup(storage_account_name, storage_account_key,
[payload_dropping_policy, credential_policy])
blob = await self._create_blob()
# Act
stream = LargeStream(LARGEST_BLOCK_SIZE)
blockId = str(uuid.uuid4())
requestId = str(uuid.uuid4())
resp = await blob.stage_block(blockId,
stream,
length=LARGEST_BLOCK_SIZE,
client_request_id=requestId)
await blob.commit_block_list([BlobBlock(blockId)])
block_list = await blob.get_block_list()
# Assert
self.assertIsNotNone(resp)
assert 'content_md5' in resp
assert 'content_crc64' in resp
assert 'request_id' in resp
self.assertIsNotNone(block_list)
self.assertEqual(len(block_list), 2)
self.assertEqual(len(block_list[1]), 0)
self.assertEqual(len(block_list[0]), 1)
self.assertEqual(payload_dropping_policy.put_block_counter, 1)
self.assertEqual(payload_dropping_policy.put_block_sizes[0],
LARGEST_BLOCK_SIZE)
def flush(self):
if self._buf:
block_id = _block_id(self._block_offset)
self._service.put_block(self._container_name, self._blob_name,
bytes(self._buf), block_id)
self._blocks.append(BlobBlock(block_id))
self._block_offset += 1
self._buf.clear()
def test_put_block_list(self, resource_group, location, storage_account, storage_account_key):
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
blob.stage_block('1', b'AAA')
blob.stage_block('2', b'BBB')
blob.stage_block('3', b'CCC')
# Act
block_list = [BlobBlock(block_id='1'), BlobBlock(block_id='2'), BlobBlock(block_id='3')]
put_block_list_resp = blob.commit_block_list(block_list)
# Assert
content = blob.download_blob()
self.assertEqual(content.readall(), b'AAABBBCCC')
self.assertEqual(content.properties.etag, put_block_list_resp.get('etag'))
self.assertEqual(content.properties.last_modified, put_block_list_resp.get('last_modified'))
def _write_to_blob(self, data):
# block_id's have to be base-64 strings normalized to have the same length.
block_id = base64.b64encode('{0:-32d}'.format(
self.block_number).encode()).decode()
self._blob_to_upload.stage_block(block_id, data)
self.block_list.append(BlobBlock(block_id))
self.block_number = self.block_number + 1
def complete_chunked_upload(self, uuid, final_path, storage_metadata):
"""
Complete the chunked upload and store the final results in the path indicated.
Returns nothing.
"""
# Commit the blob's blocks.
upload_blob_name = self._upload_blob_name_from_uuid(
uuid) # upload/<uuid>
upload_blob_path = self._upload_blob_path_from_uuid(
uuid) # storage/path/upload/<uuid>
block_list = [
BlobBlock(block_id) for block_id in storage_metadata[_BLOCKS_KEY]
]
try:
if storage_metadata[_CONTENT_TYPE_KEY] is not None:
content_settings = ContentSettings(
content_type=storage_metadata[_CONTENT_TYPE_KEY])
self._blob(upload_blob_path).commit_block_list(
block_list, content_settings=content_settings)
else:
self._blob(upload_blob_path).commit_block_list(block_list)
except AzureError:
logger.exception(
"Exception when trying to put block list for path %s from upload %s",
final_path,
uuid,
)
raise IOError("Exception when trying to put block list")
# Copy the blob to its final location.
upload_blob_name = self._upload_blob_name_from_uuid(uuid)
copy_source_url = self.get_direct_download_url(upload_blob_name,
expires_in=300)
try:
final_blob_name = self._blob_name_from_path(final_path)
cp = self._blob(final_blob_name).start_copy_from_url(
copy_source_url)
except AzureError:
logger.exception(
"Exception when trying to set copy uploaded blob %s to path %s",
uuid, final_path)
raise IOError("Exception when trying to copy uploaded blob")
self._await_copy(final_blob_name)
# Delete the original blob.
logger.debug("Deleting chunked upload %s at path %s", uuid,
upload_blob_path)
try:
self._blob(upload_blob_path).delete_blob()
except AzureError:
logger.exception(
"Exception when trying to set delete uploaded blob %s", uuid)
raise IOError("Exception when trying to delete uploaded blob")
