async def test_commit_block_list_with_tags(self, resource_group, location, storage_account, storage_account_key): await self._setup(storage_account, storage_account_key) tags = {"tag1": "firsttag", "tag2": "secondtag", "tag3": "thirdtag"} blob_client, resp = await self._create_empty_block_blob( tags={'condition tag': 'test tag'}) await blob_client.stage_block('1', b'AAA') await blob_client.stage_block('2', b'BBB') await blob_client.stage_block('3', b'CCC') # Act block_list = [ BlobBlock(block_id='1'), BlobBlock(block_id='2'), BlobBlock(block_id='3') ] with self.assertRaises(ResourceModifiedError): await blob_client.commit_block_list( block_list, tags=tags, if_tags_match_condition="\"condition tag\"='wrong tag'") await blob_client.commit_block_list( block_list, tags=tags, if_tags_match_condition="\"condition tag\"='test tag'") resp = await blob_client.get_blob_tags() # Assert self.assertIsNotNone(resp) self.assertEqual(len(resp), len(tags))
def test_put_block_and_put_block_list(self): # Arrange blob_client, _ = self._create_block_blob() blob_client.stage_block('1', b'AAA', cpk=TEST_ENCRYPTION_KEY) blob_client.stage_block('2', b'BBB', cpk=TEST_ENCRYPTION_KEY) blob_client.stage_block('3', b'CCC', cpk=TEST_ENCRYPTION_KEY) # Act block_list = [BlobBlock(block_id='1'), BlobBlock(block_id='2'), BlobBlock(block_id='3')] put_block_list_resp = blob_client.commit_block_list(block_list, cpk=TEST_ENCRYPTION_KEY) # Assert self.assertIsNotNone(put_block_list_resp['etag']) self.assertIsNotNone(put_block_list_resp['last_modified']) self.assertTrue(put_block_list_resp['request_server_encrypted']) self.assertEqual(put_block_list_resp['encryption_key_sha256'], TEST_ENCRYPTION_KEY.key_hash) # Act get the blob content without cpk should fail with self.assertRaises(HttpResponseError): blob_client.download_blob() # Act get the blob content blob = blob_client.download_blob(cpk=TEST_ENCRYPTION_KEY) # Assert content was retrieved with the cpk self.assertEqual(blob.content_as_bytes(), b'AAABBBCCC') self.assertEqual(blob.properties.etag, put_block_list_resp['etag']) self.assertEqual(blob.properties.last_modified, put_block_list_resp['last_modified']) self.assertEqual(blob.properties.encryption_key_sha256, TEST_ENCRYPTION_KEY.key_hash)
def test_put_block_from_url_and_commit(self): # Arrange dest_blob_name = self.get_resource_name('destblob') # Act part 1: make put block from url calls self.bs.put_block_from_url(self.container_name, dest_blob_name, self.source_blob_url, source_range_start=0, source_range_end=4 * 1024 - 1, block_id=1) self.bs.put_block_from_url(self.container_name, dest_blob_name, self.source_blob_url, source_range_start=4 * 1024, source_range_end=8 * 1024, block_id=2) # Assert blocks block_list = self.bs.get_block_list(self.container_name, dest_blob_name, None, 'all') self.assertEqual(len(block_list.uncommitted_blocks), 2) self.assertEqual(len(block_list.committed_blocks), 0) # Act part 2: commit the blocks block_list = [BlobBlock(id='1'), BlobBlock(id='2')] self.bs.put_block_list(self.container_name, dest_blob_name, block_list) # Assert destination blob has right content blob = self.bs.get_blob_to_bytes(self.container_name, dest_blob_name) self.assertEqual(blob.content, self.source_blob_data)
def test_get_block_list_committed_blocks(self): # Arrange blob_name = self._get_blob_reference() self.bs.put_block(self.container_name, blob_name, b'AAA', '1') self.bs.put_block(self.container_name, blob_name, b'BBB', '2') self.bs.put_block(self.container_name, blob_name, b'CCC', '3') block_list = [BlobBlock(id='1'), BlobBlock(id='2'), BlobBlock(id='3')] self.bs.put_block_list(self.container_name, blob_name, block_list) # Act block_list = self.bs.get_block_list(self.container_name, blob_name, None, 'all') # Assert self.assertIsNotNone(block_list) self.assertIsInstance(block_list, BlobBlockList) self.assertEqual(len(block_list.uncommitted_blocks), 0) self.assertEqual(len(block_list.committed_blocks), 3) self.assertEqual(block_list.committed_blocks[0].id, '1') self.assertEqual(block_list.committed_blocks[0].size, 3) self.assertEqual(block_list.committed_blocks[1].id, '2') self.assertEqual(block_list.committed_blocks[1].size, 3) self.assertEqual(block_list.committed_blocks[2].id, '3') self.assertEqual(block_list.committed_blocks[2].size, 3)
def test_get_block_list_committed_blocks(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) blob.stage_block('1', b'AAA') blob.stage_block('2', b'BBB') blob.stage_block('3', b'CCC') block_list = [BlobBlock(block_id='1'), BlobBlock(block_id='2'), BlobBlock(block_id='3')] blob.commit_block_list(block_list) # Act block_list = blob.get_block_list('committed') # Assert self.assertIsNotNone(block_list) self.assertEqual(len(block_list), 2) self.assertEqual(len(block_list[1]), 0) self.assertEqual(len(block_list[0]), 3) self.assertEqual(block_list[0][0].id, '1') self.assertEqual(block_list[0][0].size, 3) self.assertEqual(block_list[0][1].id, '2') self.assertEqual(block_list[0][1].size, 3) self.assertEqual(block_list[0][2].id, '3') self.assertEqual(block_list[0][2].size, 3)
def test_get_block_list_committed_blocks(self): # Arrange blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) blob.stage_block('1', b'AAA') blob.stage_block('2', b'BBB') blob.stage_block('3', b'CCC') block_list = [ BlobBlock(block_id='1'), BlobBlock(block_id='2'), BlobBlock(block_id='3') ] blob.commit_block_list(block_list) # Act block_list = blob.get_block_list('committed') # Assert self.assertIsNotNone(block_list) self.assertEqual(len(block_list), 2) self.assertEqual(len(block_list[1]), 0) self.assertEqual(len(block_list[0]), 3) self.assertEqual(block_list[0][0].id, '1') self.assertEqual(block_list[0][0].size, 3) self.assertEqual(block_list[0][1].id, '2') self.assertEqual(block_list[0][1].size, 3) self.assertEqual(block_list[0][2].id, '3') self.assertEqual(block_list[0][2].size, 3)
async def _test_put_block_list(self): # Arrange await self._setup() blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) await blob.stage_block('1', b'AAA') await blob.stage_block('2', b'BBB') await blob.stage_block('3', b'CCC') # Act block_list = [ BlobBlock(block_id='1'), BlobBlock(block_id='2'), BlobBlock(block_id='3') ] put_block_list_resp = await blob.commit_block_list(block_list) # Assert content = await blob.download_blob() actual = await content.readall() self.assertEqual(actual, b'AAABBBCCC') self.assertEqual(content.properties.etag, put_block_list_resp.get('etag')) self.assertEqual(content.properties.last_modified, put_block_list_resp.get('last_modified'))
async def _test_put_block_from_url_and_commit(self): # Arrange # create source blob and get source blob url source_blob_name = self.get_resource_name("sourceblob") self.config.use_byte_buffer = True # Make sure using chunk upload, then we can record the request source_blob_client, _ = await self._create_block_blob(blob_name=source_blob_name, data=self.byte_data) source_blob_sas = generate_blob_sas( source_blob_client.account_name, source_blob_client.container_name, source_blob_client.blob_name, snapshot=source_blob_client.snapshot, account_key=source_blob_client.credential.account_key, permission=BlobSasPermissions(read=True), expiry=datetime.utcnow() + timedelta(hours=1) ) source_blob_url = source_blob_client.url + "?" + source_blob_sas # create destination blob self.config.use_byte_buffer = False destination_blob_client, _ = await self._create_block_blob(cpk=TEST_ENCRYPTION_KEY) # Act part 1: make put block from url calls await destination_blob_client.stage_block_from_url(block_id=1, source_url=source_blob_url, source_offset=0, source_length=4 * 1024, cpk=TEST_ENCRYPTION_KEY) await destination_blob_client.stage_block_from_url(block_id=2, source_url=source_blob_url, source_offset=4 * 1024, source_length=4 * 1024, cpk=TEST_ENCRYPTION_KEY) # Assert blocks committed, uncommitted = await destination_blob_client.get_block_list('all') self.assertEqual(len(uncommitted), 2) self.assertEqual(len(committed), 0) # commit the blocks without cpk should fail block_list = [BlobBlock(block_id='1'), BlobBlock(block_id='2')] with self.assertRaises(HttpResponseError): await destination_blob_client.commit_block_list(block_list) # Act commit the blocks with cpk should succeed put_block_list_resp = await destination_blob_client.commit_block_list(block_list, cpk=TEST_ENCRYPTION_KEY) # Assert self.assertIsNotNone(put_block_list_resp['etag']) self.assertIsNotNone(put_block_list_resp['last_modified']) self.assertTrue(put_block_list_resp['request_server_encrypted']) self.assertEqual(put_block_list_resp['encryption_key_sha256'], TEST_ENCRYPTION_KEY.key_hash) # Act get the blob content blob = await destination_blob_client.download_blob(cpk=TEST_ENCRYPTION_KEY) # Assert content was retrieved with the cpk self.assertEqual(await blob.readall(), self.byte_data[0: 8 * 1024]) self.assertEqual(blob.properties.etag, put_block_list_resp['etag']) self.assertEqual(blob.properties.last_modified, put_block_list_resp['last_modified']) self.assertEqual(blob.properties.encryption_key_sha256, TEST_ENCRYPTION_KEY.key_hash)
def test_put_block_from_url_and_commit(self): # Arrange dest_blob_name = self._get_blob_reference() # Act part 1: make put block from url calls self.bbs.put_block_from_url(self.container_name, dest_blob_name, self.source_blob_url, source_range_start=0, source_range_end=4 * 1024 - 1, block_id=1, cpk=TEST_ENCRYPTION_KEY) self.bbs.put_block_from_url(self.container_name, dest_blob_name, self.source_blob_url, source_range_start=4 * 1024, source_range_end=8 * 1024, block_id=2, cpk=TEST_ENCRYPTION_KEY) # Assert blocks block_list = self.bbs.get_block_list(self.container_name, dest_blob_name, None, 'all') self.assertEqual(len(block_list.uncommitted_blocks), 2) self.assertEqual(len(block_list.committed_blocks), 0) # commit the blocks without cpk should fail block_list = [BlobBlock(id='1'), BlobBlock(id='2')] with self.assertRaises(AzureHttpError): self.bbs.put_block_list(self.container_name, dest_blob_name, block_list) # Act commit the blocks with cpk should succeed put_block_list_resp = self.bbs.put_block_list(self.container_name, dest_blob_name, block_list, cpk=TEST_ENCRYPTION_KEY) # Assert self.assertIsNotNone(put_block_list_resp.etag) self.assertIsNotNone(put_block_list_resp.last_modified) self.assertTrue(put_block_list_resp.request_server_encrypted) self.assertEqual(put_block_list_resp.encryption_key_sha256, TEST_ENCRYPTION_KEY.key_hash) # Assert destination blob has right content blob = self.bbs.get_blob_to_bytes(self.container_name, dest_blob_name, cpk=TEST_ENCRYPTION_KEY) self.assertEqual(blob.content, self.byte_data[0:8 * 1024 + 1]) self.assertEqual(blob.properties.etag, put_block_list_resp.etag) self.assertEqual(blob.properties.last_modified, put_block_list_resp.last_modified) self.assertEqual(blob.properties.encryption_key_sha256, TEST_ENCRYPTION_KEY.key_hash)
def test_put_block_list_with_md5(self): # Arrange blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) blob.stage_block('1', b'AAA') blob.stage_block('2', b'BBB') blob.stage_block('3', b'CCC') # Act block_list = [BlobBlock(block_id='1'), BlobBlock(block_id='2'), BlobBlock(block_id='3')] blob.commit_block_list(block_list, validate_content=True)
def test_put_block_list_with_md5(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) blob.stage_block('1', b'AAA') blob.stage_block('2', b'BBB') blob.stage_block('3', b'CCC') # Act block_list = [BlobBlock(block_id='1'), BlobBlock(block_id='2'), BlobBlock(block_id='3')] blob.commit_block_list(block_list, validate_content=True)
def test_put_block_list_with_md5(self): # Arrange blob_name = self._get_blob_reference() self.bs.put_block(self.container_name, blob_name, b'AAA', '1') self.bs.put_block(self.container_name, blob_name, b'BBB', '2') self.bs.put_block(self.container_name, blob_name, b'CCC', '3') # Act block_list = [BlobBlock(id='1'), BlobBlock(id='2'), BlobBlock(id='3')] self.bs.put_block_list(self.container_name, blob_name, block_list, validate_content=True)
async def test_put_block_and_put_block_list(self, resource_group, location, storage_account, storage_account_key): # Arrange # test chunking functionality by reducing the size of each chunk, # otherwise the tests would take too long to execute bsc = BlobServiceClient( self.account_url(storage_account.name, "blob"), storage_account_key, max_single_put_size=1024, min_large_block_upload_threshold=1024, max_block_size=1024, max_page_size=1024, transport=AiohttpTestTransport(connection_data_block_size=1024)) await self._setup(bsc) self.container_name = self.get_resource_name('utcontainer') blob_client, _ = await self._create_block_blob(bsc) await blob_client.stage_block('1', b'AAA', cpk=TEST_ENCRYPTION_KEY) await blob_client.stage_block('2', b'BBB', cpk=TEST_ENCRYPTION_KEY) await blob_client.stage_block('3', b'CCC', cpk=TEST_ENCRYPTION_KEY) # Act block_list = [ BlobBlock(block_id='1'), BlobBlock(block_id='2'), BlobBlock(block_id='3') ] put_block_list_resp = await blob_client.commit_block_list( block_list, cpk=TEST_ENCRYPTION_KEY) # Assert self.assertIsNotNone(put_block_list_resp['etag']) self.assertIsNotNone(put_block_list_resp['last_modified']) self.assertTrue(put_block_list_resp['request_server_encrypted']) self.assertEqual(put_block_list_resp['encryption_key_sha256'], TEST_ENCRYPTION_KEY.key_hash) # Act get the blob content without cpk should fail with self.assertRaises(HttpResponseError): await blob_client.download_blob() # Act get the blob content blob = await blob_client.download_blob(cpk=TEST_ENCRYPTION_KEY) # Assert content was retrieved with the cpk self.assertEqual(await blob.readall(), b'AAABBBCCC') self.assertEqual(blob.properties.etag, put_block_list_resp['etag']) self.assertEqual(blob.properties.last_modified, put_block_list_resp['last_modified']) self.assertEqual(blob.properties.encryption_key_sha256, TEST_ENCRYPTION_KEY.key_hash)
def test_put_block_and_put_block_list(self, resource_group, location, storage_account, storage_account_key): # Arrange bsc = BlobServiceClient(self.account_url(storage_account, "blob"), credential=storage_account_key, connection_data_block_size=1024, max_single_put_size=1024, min_large_block_upload_threshold=1024, max_block_size=1024, max_page_size=1024) self._setup(bsc) blob_client, _ = self._create_block_blob(bsc) blob_client.stage_block('1', b'AAA', encryption_scope=TEST_ENCRYPTION_KEY_SCOPE) blob_client.stage_block('2', b'BBB', encryption_scope=TEST_ENCRYPTION_KEY_SCOPE) blob_client.stage_block('3', b'CCC', encryption_scope=TEST_ENCRYPTION_KEY_SCOPE) # Act block_list = [ BlobBlock(block_id='1'), BlobBlock(block_id='2'), BlobBlock(block_id='3') ] put_block_list_resp = blob_client.commit_block_list( block_list, encryption_scope=TEST_ENCRYPTION_KEY_SCOPE) # Assert self.assertIsNotNone(put_block_list_resp['etag']) self.assertIsNotNone(put_block_list_resp['last_modified']) self.assertTrue(put_block_list_resp['request_server_encrypted']) self.assertEqual(put_block_list_resp['encryption_scope'], TEST_ENCRYPTION_KEY_SCOPE) # Act get the blob content blob = blob_client.download_blob() # Assert content was retrieved with the cpk self.assertEqual(blob.readall(), b'AAABBBCCC') self.assertEqual(blob.properties.etag, put_block_list_resp['etag']) self.assertEqual(blob.properties.last_modified, put_block_list_resp['last_modified']) self.assertEqual(blob.properties.encryption_scope, TEST_ENCRYPTION_KEY_SCOPE) self._teardown(bsc)
def test_put_block_list_invalid_block_id(self): # Arrange blob_name = self._get_blob_reference() self.bs.put_block(self.container_name, blob_name, b'AAA', '1') self.bs.put_block(self.container_name, blob_name, b'BBB', '2') self.bs.put_block(self.container_name, blob_name, b'CCC', '3') # Act try: block_list = [ BlobBlock(id='1'), BlobBlock(id='2'), BlobBlock(id='4')] self.bs.put_block_list(self.container_name, blob_name, block_list) self.fail() except AzureHttpError as e: self.assertGreaterEqual(str(e).find('specified block list is invalid'), 0)
def blocks(self): container_name = self._create_container() blob_name = self._get_blob_reference() # Put block # Block id's must be the same length self.service.put_block(container_name, blob_name, b'AAA', '1') self.service.put_block(container_name, blob_name, b'BBB', '2') self.service.put_block(container_name, blob_name, b'CCC', '3') # Get Block List # Defaults to committed only, specify all to get committed and uncommitted block_list = self.service.get_block_list( container_name, blob_name, block_list_type=BlockListType.All) uncommitted = len(block_list.uncommitted_blocks) # 3 committed = len(block_list.committed_blocks) # 0 # Note the blob does not yet appears as blocks have not been committed exists = self.service.exists(container_name, blob_name) # False # Commit the blocks # BlockBlock state defaults to Latest meaning the uncommitted and then # the committed list is searched for the block id to commit block_list = [BlobBlock(id='1'), BlobBlock(id='2'), BlobBlock(id='3')] self.service.put_block_list(container_name, blob_name, block_list) # Get Block List # Defaults to committed only, specify all to get committed and uncommitted block_list = self.service.get_block_list( container_name, blob_name, block_list_type=BlockListType.All) uncommitted = len(block_list.uncommitted_blocks) # 0 committed = len(block_list.committed_blocks) # 3 # Add a block # Put the block self.service.put_block(container_name, blob_name, b'DDD', '4') # Get the existing blocks block_list = self.service.get_block_list( container_name, blob_name, block_list_type=BlockListType.All) uncommitted = len(block_list.uncommitted_blocks) # 1 committed = len(block_list.committed_blocks) # 3 # Added the new block to the existing list and commit new_block_list = block_list.committed_blocks new_block_list.append(block_list.uncommitted_blocks[0]) self.service.put_block_list(container_name, blob_name, new_block_list) self.service.delete_container(container_name)
def test_put_block_list(self): # Arrange blob_name = self._get_blob_reference() self.bs.put_block(self.container_name, blob_name, b'AAA', '1') self.bs.put_block(self.container_name, blob_name, b'BBB', '2') self.bs.put_block(self.container_name, blob_name, b'CCC', '3') # Act block_list = [BlobBlock(id='1'), BlobBlock(id='2'), BlobBlock(id='3')] self.bs.put_block_list(self.container_name, blob_name, block_list) # Assert blob = self.bs.get_blob_to_bytes(self.container_name, blob_name) self.assertEqual(blob.content, b'AAABBBCCC')
def test_put_block_and_put_block_list(self): # Arrange blob_name = self._get_blob_reference() self.bbs.put_block(self.container_name, blob_name, b'AAA', '1', cpk=TEST_ENCRYPTION_KEY) self.bbs.put_block(self.container_name, blob_name, b'BBB', '2', cpk=TEST_ENCRYPTION_KEY) self.bbs.put_block(self.container_name, blob_name, b'CCC', '3', cpk=TEST_ENCRYPTION_KEY) # Act block_list = [BlobBlock(id='1'), BlobBlock(id='2'), BlobBlock(id='3')] put_block_list_resp = self.bbs.put_block_list(self.container_name, blob_name, block_list, cpk=TEST_ENCRYPTION_KEY) # Assert self.assertIsNotNone(put_block_list_resp.etag) self.assertIsNotNone(put_block_list_resp.last_modified) self.assertTrue(put_block_list_resp.request_server_encrypted) self.assertEqual(put_block_list_resp.encryption_key_sha256, TEST_ENCRYPTION_KEY.key_hash) # Act get the blob content without cpk should fail with self.assertRaises(AzureHttpError): self.bbs.get_blob_to_bytes(self.container_name, blob_name) # Act get the blob content blob = self.bbs.get_blob_to_bytes(self.container_name, blob_name, cpk=TEST_ENCRYPTION_KEY) # Assert content was retrieved with the cpk self.assertEqual(blob.content, b'AAABBBCCC') self.assertEqual(blob.properties.etag, put_block_list_resp.etag) self.assertEqual(blob.properties.last_modified, put_block_list_resp.last_modified) self.assertEqual(blob.properties.encryption_key_sha256, TEST_ENCRYPTION_KEY.key_hash)
def test_put_block_list_invalid_block_id(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) blob.stage_block('1', b'AAA') blob.stage_block('2', b'BBB') blob.stage_block('3', b'CCC') # Act try: block_list = [BlobBlock(block_id='1'), BlobBlock(block_id='2'), BlobBlock(block_id='4')] blob.commit_block_list(block_list) self.fail() except HttpResponseError as e: self.assertGreaterEqual(str(e).find('specified block list is invalid'), 0)
async def test_put_block_stream_largest(self, storage_account_name, storage_account_key): await self._setup(storage_account_name, storage_account_key) blob = await self._create_blob() # Act stream = LargeStream(LARGEST_BLOCK_SIZE) blockId = str(uuid.uuid4()) requestId = str(uuid.uuid4()) resp = await blob.stage_block(blockId, stream, length=LARGEST_BLOCK_SIZE, client_request_id=requestId) await blob.commit_block_list([BlobBlock(blockId)]) block_list = await blob.get_block_list() # Assert self.assertIsNotNone(resp) assert 'content_md5' in resp assert 'content_crc64' in resp assert 'request_id' in resp self.assertIsNotNone(block_list) self.assertEqual(len(block_list), 2) self.assertEqual(len(block_list[1]), 0) self.assertEqual(len(block_list[0]), 1) self.assertEqual(block_list[0][0].size, LARGEST_BLOCK_SIZE)
def __copy(args): s3cli = boto3.resource("s3", aws_access_key_id=args.source_secret_id, aws_secret_access_key=args.source_secret_key) azblob = BlockBlobService(args.destination_account, args.destination_SAS) s3object = s3cli.Object(args.source_bucket, args.source_file) print("Opening S3 object {0}/{1}".format(args.source_bucket, args.source_file)) chunk = s3object.get(PartNumber=1) nchunks = chunk['PartsCount'] blocks = [] for x in range(1, nchunks + 1): chunk = s3object.get(PartNumber=x) print("Reading part {0}/{1}".format(x, nchunks)) part = chunk['Body'].read() print("Writing part {0}/{1}. Size: {2} bytes".format( x, nchunks, len(part))) blockid = str(uuid.uuid4()) azblob.put_block(args.destination_container, args.destination_file, part, blockid) blocks.append(BlobBlock(id=blockid)) print("Committing file {0}/{1}".format(args.destination_container, args.destination_file)) azblob.put_block_list(args.destination_container, args.destination_file, blocks) print("Committed")
async def test_put_block_bytes_largest_without_network(self, resource_group, location, storage_account, storage_account_key): payload_dropping_policy = PayloadDroppingPolicy() credential_policy = _format_shared_key_credential(storage_account.name, storage_account_key) await self._setup(storage_account, storage_account_key, [payload_dropping_policy, credential_policy]) blob = await self._create_blob() # Act data = urandom(LARGEST_BLOCK_SIZE) blockId = str(uuid.uuid4()).encode('utf-8') resp = await blob.stage_block( blockId, data, length=LARGEST_BLOCK_SIZE) await blob.commit_block_list([BlobBlock(blockId)]) block_list = await blob.get_block_list() # Assert self.assertIsNotNone(resp) assert 'content_md5' in resp assert 'content_crc64' in resp assert 'request_id' in resp self.assertIsNotNone(block_list) self.assertEqual(len(block_list), 2) self.assertEqual(len(block_list[1]), 0) self.assertEqual(len(block_list[0]), 1) self.assertEqual(payload_dropping_policy.put_block_counter, 1) self.assertEqual(payload_dropping_policy.put_block_sizes[0], LARGEST_BLOCK_SIZE)
def parallel_chunky_upload(self, container_name, full_blob_name, data, chunks=5): debug = False threads = [] block_ids = [] chunk_size = len(data) / chunks chunks = [ data[i:i + chunk_size] for i in xrange(0, len(data), chunk_size) ] for chunk in chunks: uid = self.generate_uid() block_ids.append(BlobBlock(id=uid)) t = threading.Thread(target=self._upload_block, args=( container_name, full_blob_name, chunk, uid, )) threads.append(t) t.start() [t.join() for t in threads] self.service.put_block_list(container_name, full_blob_name, block_ids) return full_blob_name
def test_retry_put_block_with_non_seekable_stream(self): if TestMode.need_recording_file(self.test_mode): return # Arrange blob_name = self.get_resource_name('blob') data = self.get_random_bytes(PUT_BLOCK_SIZE) data_stream = self.NonSeekableStream(BytesIO(data)) # rig the response so that it fails for a single time self.bs.response_callback = ResponseCallback(status=201, new_status=408).override_first_status self.bs.retry = ExponentialRetry(initial_backoff=1, increment_base=2, max_attempts=3).retry # Act # Note: put_block transforms non-seekable streams into byte arrays before handing it off to the executor self.bs.put_block(self.container_name, blob_name, data_stream, 1) # Assert block_list = self.bs.get_block_list(self.container_name, blob_name, block_list_type="uncommitted") self.assertEqual(len(block_list.uncommitted_blocks), 1) self.assertEqual(block_list.uncommitted_blocks[0].size, PUT_BLOCK_SIZE) # Commit block and verify content block_list = [BlobBlock(id='1')] self.bs.put_block_list(self.container_name, blob_name, block_list) # Assert blob = self.bs.get_blob_to_bytes(self.container_name, blob_name) self.assertEqual(blob.content, data)
def save_to_blob(local_path: str, remote_name: str, container_name: str) -> None: # Instantiate a new BlobServiceClient using a connection string blob_service_client = BlobServiceClient.from_connection_string( connection_string) # Instantiate a new ContainerClient container_client = blob_service_client.get_container_client(container_name) # Ensure container is created # container_client.create_container() # Instantiate a new BlobClient blob_client = container_client.get_blob_client(remote_name) # Upload content to block blob # with open(local_path, "rb") as data: # blob_client.upload_blob(data, timeout=600) # TODO do something like this instead block_list = [] chunk_size = 4194304 with open(local_path, "rb") as data: while True: read_data = data.read(chunk_size) if not read_data: break blk_id = uuid_to_str(uuid.uuid4()) blob_client.stage_block(block_id=blk_id, data=read_data) block_list.append(BlobBlock(block_id=blk_id)) blob_client.commit_block_list(block_list)
async def test_put_block_stream_largest_without_network( self, storage_account_name, storage_account_key): payload_dropping_policy = PayloadDroppingPolicy() credential_policy = _format_shared_key_credential( storage_account_name, storage_account_key) await self._setup(storage_account_name, storage_account_key, [payload_dropping_policy, credential_policy]) blob = await self._create_blob() # Act stream = LargeStream(LARGEST_BLOCK_SIZE) blockId = str(uuid.uuid4()) requestId = str(uuid.uuid4()) resp = await blob.stage_block(blockId, stream, length=LARGEST_BLOCK_SIZE, client_request_id=requestId) await blob.commit_block_list([BlobBlock(blockId)]) block_list = await blob.get_block_list() # Assert self.assertIsNotNone(resp) assert 'content_md5' in resp assert 'content_crc64' in resp assert 'request_id' in resp self.assertIsNotNone(block_list) self.assertEqual(len(block_list), 2) self.assertEqual(len(block_list[1]), 0) self.assertEqual(len(block_list[0]), 1) self.assertEqual(payload_dropping_policy.put_block_counter, 1) self.assertEqual(payload_dropping_policy.put_block_sizes[0], LARGEST_BLOCK_SIZE)
def flush(self): if self._buf: block_id = _block_id(self._block_offset) self._service.put_block(self._container_name, self._blob_name, bytes(self._buf), block_id) self._blocks.append(BlobBlock(block_id)) self._block_offset += 1 self._buf.clear()
def test_put_block_list(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) blob.stage_block('1', b'AAA') blob.stage_block('2', b'BBB') blob.stage_block('3', b'CCC') # Act block_list = [BlobBlock(block_id='1'), BlobBlock(block_id='2'), BlobBlock(block_id='3')] put_block_list_resp = blob.commit_block_list(block_list) # Assert content = blob.download_blob() self.assertEqual(content.readall(), b'AAABBBCCC') self.assertEqual(content.properties.etag, put_block_list_resp.get('etag')) self.assertEqual(content.properties.last_modified, put_block_list_resp.get('last_modified'))
def _write_to_blob(self, data): # block_id's have to be base-64 strings normalized to have the same length. block_id = base64.b64encode('{0:-32d}'.format( self.block_number).encode()).decode() self._blob_to_upload.stage_block(block_id, data) self.block_list.append(BlobBlock(block_id)) self.block_number = self.block_number + 1
def complete_chunked_upload(self, uuid, final_path, storage_metadata): """ Complete the chunked upload and store the final results in the path indicated. Returns nothing. """ # Commit the blob's blocks. upload_blob_name = self._upload_blob_name_from_uuid( uuid) # upload/<uuid> upload_blob_path = self._upload_blob_path_from_uuid( uuid) # storage/path/upload/<uuid> block_list = [ BlobBlock(block_id) for block_id in storage_metadata[_BLOCKS_KEY] ] try: if storage_metadata[_CONTENT_TYPE_KEY] is not None: content_settings = ContentSettings( content_type=storage_metadata[_CONTENT_TYPE_KEY]) self._blob(upload_blob_path).commit_block_list( block_list, content_settings=content_settings) else: self._blob(upload_blob_path).commit_block_list(block_list) except AzureError: logger.exception( "Exception when trying to put block list for path %s from upload %s", final_path, uuid, ) raise IOError("Exception when trying to put block list") # Copy the blob to its final location. upload_blob_name = self._upload_blob_name_from_uuid(uuid) copy_source_url = self.get_direct_download_url(upload_blob_name, expires_in=300) try: final_blob_name = self._blob_name_from_path(final_path) cp = self._blob(final_blob_name).start_copy_from_url( copy_source_url) except AzureError: logger.exception( "Exception when trying to set copy uploaded blob %s to path %s", uuid, final_path) raise IOError("Exception when trying to copy uploaded blob") self._await_copy(final_blob_name) # Delete the original blob. logger.debug("Deleting chunked upload %s at path %s", uuid, upload_blob_path) try: self._blob(upload_blob_path).delete_blob() except AzureError: logger.exception( "Exception when trying to set delete uploaded blob %s", uuid) raise IOError("Exception when trying to delete uploaded blob")