def upload_third_party(self): logger.info("uploading third-party tools from %s", self.third_party) account_name = self.results["deploy"]["fuzz-name"]["value"] key = self.results["deploy"]["fuzz-key"]["value"] account_url = "https://%s.blob.core.windows.net" % account_name client = BlobServiceClient(account_url, credential=key) containers = [x["name"] for x in client.list_containers()] for name in os.listdir(self.third_party): path = os.path.join(self.third_party, name) if not os.path.isdir(path): continue if name not in containers: client.create_container(name) expiry = datetime.utcnow() + timedelta(minutes=30) sas = generate_container_sas( account_name, name, account_key=key, permission=ContainerSasPermissions( read=True, write=True, delete=True, list=True ), expiry=expiry, ) url = "%s/%s?%s" % (account_url, name, sas) subprocess.check_output( [self.azcopy, "sync", path, url, "--delete-destination", "true"] )
def upload_instance_setup(self): logger.info("uploading instance-specific-setup from %s", self.instance_specific) account_name = self.results["deploy"]["func-name"]["value"] key = self.results["deploy"]["func-key"]["value"] account_url = "https://%s.blob.core.windows.net" % account_name client = BlobServiceClient(account_url, credential=key) if "instance-specific-setup" not in [ x["name"] for x in client.list_containers() ]: client.create_container("instance-specific-setup") expiry = datetime.utcnow() + timedelta(minutes=30) sas = generate_container_sas( account_name, "instance-specific-setup", account_key=key, permission=ContainerSasPermissions( read=True, write=True, delete=True, list=True ), expiry=expiry, ) url = "%s/%s?%s" % (account_url, "instance-specific-setup", sas) subprocess.check_output( [ self.azcopy, "sync", self.instance_specific, url, "--delete-destination", "true", ] )
def azblob_file(azblob_credentials, cloud_bucket_name, download_gcs_public_data, public=False): acc_url = f"https://{azblob_credentials['storage_account']}.blob.core.windows.net" azblob_client = BlobServiceClient( account_url=acc_url, credential=azblob_credentials["shared_key"]) container_name = cloud_bucket_name + random_char(3).lower() if public: container_name += "public" print(f"\nUpload dataset to private azure blob container {container_name}") if container_name not in [ cntr["name"] for cntr in azblob_client.list_containers() ]: if public: azblob_client.create_container(name=container_name, metadata=None, public_access="container") else: azblob_client.create_container(name=container_name, metadata=None, public_access=None) blob_client = azblob_client.get_blob_client(container_name, "myfile.csv") with open(download_gcs_public_data, "r") as f: blob_client.upload_blob(f.read(), blob_type="BlockBlob", overwrite=True) yield f"{container_name}/myfile.csv" azblob_client.delete_container(container_name) print( f"\nAzure Blob Container {container_name} is now marked for deletion")
def createStorageContainer(storage_name, storage_key, container_names): ''' Create storage containers in the provided storage account. If the containers exist, they will not be altered. PARAMS: storage_name : string : Name of the Azure Storage Account storage_key : string : Access Key to the Azure Storage Account container_names : list[string] : List of container names to create. RETURNS: Nothing ''' blob_service = BlobServiceClient(account_url="https://" + storage_name + ".blob.core.windows.net/", credential=storage_key) for container in container_names: try: blob_service.create_container(container) print("Storage container created - ", storage_name, " : ", container) except ResourceExistsError: print("Storage container already exists - ", storage_name, " : ", container)
class AzurePersistor(Persistor): """Store models on Azure""" def __init__( self, azure_container: Text, azure_account_name: Text, azure_account_key: Text ) -> None: from azure.storage.blob import BlobServiceClient super().__init__() self.blob_service = BlobServiceClient( account_url=f"https://{azure_account_name}.blob.core.windows.net/", credential=azure_account_key, ) self._ensure_container_exists(azure_container) self.container_name = azure_container def _ensure_container_exists(self, container_name: Text) -> None: from azure.core.exceptions import ResourceExistsError try: self.blob_service.create_container(container_name) except ResourceExistsError: # no need to create the container, it already exists pass def _container_client(self) -> "ContainerClient": return self.blob_service.get_container_client(self.container_name) def list_models(self) -> List[Text]: """Lists models on remote storage. Returns: Paths to found models. """ try: blob_iterator = self._container_client().list_blobs() return [ self._model_dir_and_model_from_filename(b.name)[1] for b in blob_iterator ] except Exception as e: logger.warning(f"Failed to list models azure blob storage. {e}") return [] def _persist_tar(self, file_key: Text, tar_path: Text) -> None: """Uploads a model persisted in the `target_dir` to Azure.""" with open(tar_path, "rb") as data: self._container_client().upload_blob(name=file_key, data=data) def _retrieve_tar(self, target_filename: Text) -> None: """Downloads a model that has previously been persisted to Azure.""" blob_client = self._container_client().get_blob_client(target_filename) with open(target_filename, "wb") as blob: download_stream = blob_client.download_blob() blob.write(download_stream.readall())
def create_dls(wait_a_sec): credential = CustomTokenCredential() blob_service_client = BlobServiceClient(STORAGE_URL, credential) try: blob_service_client.delete_container("dls") except Exception: pass blob_service_client.create_container("dls") dummy_data = pd.DataFrame.from_dict({ "key0": ["val00", "val01"], "key1": ["val10", "val11"] }).to_dict(orient="split") for (data_set, file) in itertools.product(["deep", "shallow"], ["systems", "complexes", "elements"]): upload( blob_service_client, dummy_data, f"{FOLDER_NAME}/{data_set}/{file}.json", ) yield dummy_data blob_service_client.delete_container("dls")
def create_storage_account_container_if_not_exists( storage_account_client: blob.BlobServiceClient, container_name: str) -> None: try: storage_account_client.create_container(container_name) except ResourceExistsError: pass # Container already exists, silently ignore
def add_log_export(self) -> None: if not self.export_appinsights: logger.info("not exporting appinsights") return container_name = "app-insights" logger.info("adding appinsight log export") account_name = self.results["deploy"]["func_name"]["value"] key = self.results["deploy"]["func_key"]["value"] account_url = "https://%s.blob.core.windows.net" % account_name client = BlobServiceClient(account_url, credential=key) if container_name not in [x["name"] for x in client.list_containers()]: client.create_container(container_name) expiry = datetime.utcnow() + timedelta(days=2 * 365) # NOTE: as this is a long-lived SAS url, it should not be logged and only # used in the the later-on export_configurations.create() call sas = generate_container_sas( account_name, container_name, account_key=key, permission=ContainerSasPermissions(write=True), expiry=expiry, ) url = "%s/%s?%s" % (account_url, container_name, sas) record_types = ( "Requests, Event, Exceptions, Metrics, PageViews, " "PageViewPerformance, Rdd, PerformanceCounters, Availability") req = ApplicationInsightsComponentExportRequest( record_types=record_types, destination_type="Blob", is_enabled="true", destination_address=url, ) credential = AzureCliCredential() app_insight_client = ApplicationInsightsManagementClient( credential, subscription_id=self.get_subscription_id(), ) to_delete = [] for entry in app_insight_client.export_configurations.list( self.resource_group, self.application_name): if (entry.storage_name == account_name and entry.container_name == container_name): to_delete.append(entry.export_id) for export_id in to_delete: logger.info("replacing existing export: %s", export_id) app_insight_client.export_configurations.delete( self.resource_group, self.application_name, export_id) app_insight_client.export_configurations.create( self.resource_group, self.application_name, req)
def create_resource(self, name, **kwargs): if self.is_live: storage_account = kwargs.pop("storage_account") storage_account_key = kwargs.pop("storage_account_key") sas_token = generate_account_sas( account_name=storage_account.name, account_key=storage_account_key, resource_types=ResourceTypes(container=True, object=True), permission=AccountSasPermissions(create=True, list=True, write=True, read=True, add=True, delete=True, delete_previous_version=True), expiry=datetime.utcnow() + timedelta(minutes=5), ) blob_client = BlobServiceClient( storage_account.primary_endpoints.blob, sas_token) container = blob_client.create_container(name) container_uri = storage_account.primary_endpoints.blob + container.container_name self.test_class_instance.scrubber.register_name_pair( sas_token, "redacted") self.test_class_instance.scrubber.register_name_pair( container_uri, "https://storage/container") else: sas_token = "fake-sas" container_uri = "https://storage/container" return {"container_uri": container_uri, "sas_token": sas_token}
def test_user_delegation_sas_for_container(self): # SAS URL is calculated from storage key, so this test runs live only pytest.skip("Current Framework Cannot Support OAUTH") if TestMode.need_recording_file(self.test_mode): return # Arrange token_credential = self.generate_oauth_token() service_client = BlobServiceClient(self._get_oauth_account_url(), credential=token_credential) user_delegation_key = service_client.get_user_delegation_key( datetime.utcnow(), datetime.utcnow() + timedelta(hours=1)) container_client = service_client.create_container( self.get_resource_name('oauthcontainer')) token = container_client.generate_shared_access_signature( expiry=datetime.utcnow() + timedelta(hours=1), permission=ContainerPermissions.READ, user_delegation_key=user_delegation_key, account_name='emilydevtest') blob_client = container_client.get_blob_client( self.get_resource_name('oauthblob')) blob_content = self.get_random_text_data(1024) blob_client.upload_blob(blob_content, length=len(blob_content)) # Act new_blob_client = BlobClient(blob_client.url, credential=token) content = new_blob_client.download_blob() # Assert self.assertEqual(blob_content, b"".join(list(content)).decode('utf-8'))
class BlobText(): def __init__(self, account_name, account_key, sas_token, container_name): self.account_name = account_name self.account_key = account_key self.sas_token = sas_token self.container_name = container_name # Create the BlobServiceClient object which will be used to create a container client self.blob_service_client = BlobServiceClient( "https://" + account_name + ".blob.core.windows.net/", account_key) # Create the container self.container_client = self.blob_service_client.get_container_client( container_name) if self.container_client is None: self.container_client = self.blob_service_client.create_container( container_name) def write_text(self, blob_name, text): data = text.encode('utf-16') blobClient = self.container_client.get_blob_client(blob_name) if blobClient.exists: blobClient.delete_blob blobClient.upload_blob(data, overwrite=True) def read_text(self, blob_name): blobClient = self.container_client.get_blob_client(blob_name) if blobClient.exists: data = blobClient.download_blob().readall() return data.decode('utf-16') else: return None
def __init__(self, blob_service: BlobServiceClient, db: DatabaseProxy): try: self.blob_container = blob_service.create_container("images") except ResourceExistsError as exc: self.blob_container = blob_service.get_container_client( container="images") self.cosmos_container = db.create_container_if_not_exists( id="images", partition_key=PartitionKey(path="/image/path"))
class AzurePersistor(Persistor): """Store models on Azure""" def __init__( self, azure_container: Text, azure_account_name: Text, azure_account_key: Text ) -> None: from azure.storage.blob import BlobServiceClient super().__init__() self.blob_service = BlobServiceClient( account_url=f"https://{azure_account_name}.blob.core.windows.net/", credential=azure_account_key, ) self._ensure_container_exists(azure_container) self.container_name = azure_container def _ensure_container_exists(self, container_name: Text) -> None: from azure.core.exceptions import ResourceExistsError try: self.blob_service.create_container(container_name) except ResourceExistsError: # no need to create the container, it already exists pass def _container_client(self) -> "ContainerClient": return self.blob_service.get_container_client(self.container_name) def _persist_tar(self, file_key: Text, tar_path: Text) -> None: """Uploads a model persisted in the `target_dir` to Azure.""" with open(tar_path, "rb") as data: self._container_client().upload_blob(name=file_key, data=data) def _retrieve_tar(self, target_filename: Text) -> None: """Downloads a model that has previously been persisted to Azure.""" blob_client = self._container_client().get_blob_client(target_filename) with open(target_filename, "wb") as blob: download_stream = blob_client.download_blob() blob.write(download_stream.readall())
def save_to_blob(self, connection_string: str, container: str, path: str = None) -> str: """ Saves the model to an azure storage blob. The file will be located at the container and path parameter if provided, otherwise, in the root of the container.""" path = path or 'azureSDKTrackClassifier_{}_{}.model'.format( self._language, self._service) if 'sig=' in connection_string and 'AccountKey=' not in connection_string: # SAS signature. service_client = BlobServiceClient(connection_string) else: service_client = BlobServiceClient.from_connection_string( conn_str=connection_string) try: service_client.create_container(container) except ResourceExistsError: pass blob_client = service_client.get_blob_client(container, path) blob_client.upload_blob(pickle.dumps(self), overwrite=True) return path
def add_instance_id(self) -> None: logger.info("setting instance_id log export") container_name = "base-config" blob_name = "instance_id" account_name = self.results["deploy"]["func-name"]["value"] key = self.results["deploy"]["func-key"]["value"] account_url = "https://%s.blob.core.windows.net" % account_name client = BlobServiceClient(account_url, credential=key) if container_name not in [x["name"] for x in client.list_containers()]: client.create_container(container_name) blob_client = client.get_blob_client(container_name, blob_name) if blob_client.exists(): logger.debug("instance_id already exists") instance_id = uuid.UUID(blob_client.download_blob().readall().decode()) else: logger.debug("creating new instance_id") instance_id = uuid.uuid4() blob_client.upload_blob(str(instance_id)) logger.info("instance_id: %s", instance_id)
def get_container(blob_service_client: BlobServiceClient, container_name: str) -> bool: """ Get a container client """ logging.info('blob_service_client.list_containers()') logging.info(list(blob_service_client.list_containers())) try: _ = blob_service_client.get_container_client(container_name) container_client = blob_service_client.get_container_client( container_name) except: container_client = blob_service_client.create_container(container_name) return container_client
def upload_tools(self) -> None: logger.info("uploading tools from %s", self.tools) account_name = self.results["deploy"]["func-name"]["value"] key = self.results["deploy"]["func-key"]["value"] account_url = "https://%s.blob.core.windows.net" % account_name client = BlobServiceClient(account_url, credential=key) if "tools" not in [x["name"] for x in client.list_containers()]: client.create_container("tools") expiry = datetime.utcnow() + timedelta(minutes=30) sas = generate_container_sas( account_name, "tools", account_key=key, permission=ContainerSasPermissions(read=True, write=True, delete=True, list=True), expiry=expiry, ) url = "%s/%s?%s" % (account_url, "tools", sas) subprocess.check_output([ self.azcopy, "copy", os.path.join(self.tools, "*"), url, "--overwrite=true", "--recursive=true", ]) subprocess.check_output([ self.azcopy, "sync", self.tools, url, "--delete-destination", "true" ])
def test_create_container_with_default_cpk_n_deny_override( self, resource_group, location, storage_account, storage_account_key): # Arrange bsc = BlobServiceClient(self.account_url(storage_account, "blob"), credential=storage_account_key, connection_data_block_size=1024, max_single_put_size=1024, min_large_block_upload_threshold=1024, max_block_size=1024, max_page_size=1024) container_client = bsc.create_container( 'denyoverridecpkcontainer', container_encryption_scope= TEST_CONTAINER_ENCRYPTION_KEY_SCOPE_DENY_OVERRIDE) container_props = container_client.get_container_properties() self.assertEqual( container_props.encryption_scope.default_encryption_scope, TEST_CONTAINER_ENCRYPTION_KEY_SCOPE.default_encryption_scope) self.assertEqual( container_props.encryption_scope.prevent_encryption_scope_override, True) for container in bsc.list_containers( name_starts_with='denyoverridecpkcontainer'): self.assertEqual( container_props.encryption_scope.default_encryption_scope, TEST_CONTAINER_ENCRYPTION_KEY_SCOPE.default_encryption_scope) self.assertEqual( container_props.encryption_scope. prevent_encryption_scope_override, True) blob_client = container_client.get_blob_client("appendblob") # It's not allowed to set encryption scope on the blob when the container denies encryption scope override. with self.assertRaises(HttpResponseError): blob_client.upload_blob(b'aaaa', BlobType.AppendBlob, encryption_scope=TEST_ENCRYPTION_KEY_SCOPE) resp = blob_client.upload_blob(b'aaaa', BlobType.AppendBlob) self.assertEqual( resp['encryption_scope'], TEST_CONTAINER_ENCRYPTION_KEY_SCOPE.default_encryption_scope) container_client.delete_container()
def test_request_callback_signed_header(self, resource_group, location, storage_account, storage_account_key): # Arrange service = BlobServiceClient(self._account_url(storage_account.name), credential=storage_account_key) name = self.get_resource_name('cont') # Act def callback(request): if request.http_request.method == 'PUT': request.http_request.headers['x-ms-meta-hello'] = 'world' # Assert try: container = service.create_container(name, raw_request_hook=callback) metadata = container.get_container_properties().metadata self.assertEqual(metadata, {'hello': 'world'}) finally: service.delete_container(name)
def main(mytimer: func.TimerRequest) -> None: storageName = os.getenv("STORAGE_ACCOUNT_NAME") containerName = os.getenv("BLOB_CONTAINER_NAME") logging.info(f"params: {storageName}, {containerName}") ts = datetime.datetime.now() iso_ts = ts.isoformat() date_str = ts.strftime("%Y%m%d") credential = DefaultAzureCredential() oauth_url = f"https://{storageName}.blob.core.windows.net" bsc = BlobServiceClient(account_url=oauth_url, credential=credential) containerClient = bsc.get_container_client(containerName) tickers = ["BTC-USD", "MSFT"] n = len(tickers) for i in range(n): block = ",\n" ticker = tickers[i] logging.info(f"storing {ticker}") blobName = createBlobName(ticker, date_str) logging.info(f"container: {containerName}, blob: {blobName}") blobClient = containerClient.get_blob_client(blobName) try: if not blobClient.exists(): blobClient.create_append_blob() block = "" except ResourceNotFoundError: logging.info(f"Creating container: {containerName}") containerClient = bsc.create_container(containerName) blobClient = containerClient.get_blob_client(blobName) blobClient.create_append_blob() block += json.dumps({"ts": iso_ts, "v": getCurrClose(ticker)}) blobClient.append_block(block) if i<n-1: time.sleep(1)
def test_create_container_with_default_cpk_n(self, resource_group, location, storage_account, storage_account_key): # Arrange bsc = BlobServiceClient(self.account_url(storage_account, "blob"), credential=storage_account_key, connection_data_block_size=1024, max_single_put_size=1024, min_large_block_upload_threshold=1024, max_block_size=1024, max_page_size=1024) container_client = bsc.create_container( 'cpkcontainer', container_encryption_scope=TEST_CONTAINER_ENCRYPTION_KEY_SCOPE) container_props = container_client.get_container_properties() self.assertEqual( container_props.encryption_scope.default_encryption_scope, TEST_CONTAINER_ENCRYPTION_KEY_SCOPE.default_encryption_scope) self.assertEqual( container_props.encryption_scope.prevent_encryption_scope_override, False) for container in bsc.list_containers(name_starts_with='cpkcontainer'): self.assertEqual( container_props.encryption_scope.default_encryption_scope, TEST_CONTAINER_ENCRYPTION_KEY_SCOPE.default_encryption_scope) self.assertEqual( container_props.encryption_scope. prevent_encryption_scope_override, False) blob_client = container_client.get_blob_client("appendblob") # providing encryption scope when upload the blob resp = blob_client.upload_blob( b'aaaa', BlobType.AppendBlob, encryption_scope=TEST_ENCRYPTION_KEY_SCOPE) # Use the provided encryption scope on the blob self.assertEqual(resp['encryption_scope'], TEST_ENCRYPTION_KEY_SCOPE) container_client.delete_container()
def create_writable_container_sas(self, account_name, account_key, container_name, access_duration_hrs): account_url = "https://{}.blob.core.windows.net".format(account_name) blob_service_client = BlobServiceClient(account_url=account_url, credential=account_key) container_client = blob_service_client.create_container(container_name) sas_permissions = ContainerSasPermissions(read=True, write=True, delete=False, list=True) expiration = datetime.utcnow() + timedelta(hours=access_duration_hrs) sas_token = generate_container_sas(account_name, container_name, account_key=account_key, permission=sas_permissions, expiry=expiration) return '{}/{}?{}'.format(account_url, container_name, sas_token)
class AzureBlobFileSystem(AbstractFileSystem): """ Access Azure Datalake Gen2 and Azure Storage if it were a file system using Multiprotocol Access Parameters ---------- account_name: str The storage account name. This is used to authenticate requests signed with an account key and to construct the storage endpoint. It is required unless a connection string is given, or if a custom domain is used with anonymous authentication. account_key: str The storage account key. This is used for shared key authentication. If any of account key, sas token or client_id is specified, anonymous access will be used. sas_token: str A shared access signature token to use to authenticate requests instead of the account key. If account key and sas token are both specified, account key will be used to sign. If any of account key, sas token or client_id are specified, anonymous access will be used. request_session: Session The session object to use for http requests. connection_string: str If specified, this will override all other parameters besides request session. See http://azure.microsoft.com/en-us/documentation/articles/storage-configure-connection-string/ for the connection string format. socket_timeout: int If specified, this will override the default socket timeout. The timeout specified is in seconds. See DEFAULT_SOCKET_TIMEOUT in _constants.py for the default value. token_credential: TokenCredential A token credential used to authenticate HTTPS requests. The token value should be updated before its expiration. blocksize: int The block size to use for download/upload operations. Defaults to the value of ``BlockBlobService.MAX_BLOCK_SIZE`` client_id: str Client ID to use when authenticating using an AD Service Principal client/secret. client_secret: str Client secret to use when authenticating using an AD Service Principal client/secret. tenant_id: str Tenant ID to use when authenticating using an AD Service Principal client/secret. Examples -------- >>> abfs = AzureBlobFileSystem(account_name="XXXX", account_key="XXXX", container_name="XXXX") >>> abfs.ls('') ** Sharded Parquet & csv files can be read as: ** ------------------------------------------ ddf = dd.read_csv('abfs://container_name/folder/*.csv', storage_options={ ... 'account_name': ACCOUNT_NAME, 'account_key': ACCOUNT_KEY}) ddf = dd.read_parquet('abfs://container_name/folder.parquet', storage_options={ ... 'account_name': ACCOUNT_NAME, 'account_key': ACCOUNT_KEY,}) """ protocol = "abfs" def __init__( self, account_name: str, account_key: str = None, connection_string: str = None, credential: str = None, sas_token: str = None, request_session=None, socket_timeout: int = None, token_credential=None, blocksize: int = create_configuration(storage_sdk="blob").max_block_size, client_id: str = None, client_secret: str = None, tenant_id: str = None, ): AbstractFileSystem.__init__(self) self.account_name = account_name self.account_key = account_key self.connection_string = connection_string self.credential = credential self.sas_token = sas_token self.request_session = request_session self.socket_timeout = socket_timeout self.token_credential = token_credential self.blocksize = blocksize self.client_id = client_id self.client_secret = client_secret self.tenant_id = tenant_id if ( self.token_credential is None and self.account_key is None and self.sas_token is None and self.client_id is not None ): self.token_credential = self._get_token_from_service_principal() self.do_connect() @classmethod def _strip_protocol(cls, path: str): """ Remove the protocol from the input path Parameters ---------- path: str Path to remove the protocol from Returns ------- str Returns a path without the protocol """ logging.debug(f"_strip_protocol for {path}") ops = infer_storage_options(path) # we need to make sure that the path retains # the format {host}/{path} # here host is the container_name if ops.get("host", None): ops["path"] = ops["host"] + ops["path"] ops["path"] = ops["path"].lstrip("/") logging.debug(f"_strip_protocol({path}) = {ops}") return ops["path"] def _get_token_from_service_principal(self): """ Create a TokenCredential given a client_id, client_secret and tenant_id Returns ------- TokenCredential """ from azure.common.credentials import ServicePrincipalCredentials from azure.storage.common import TokenCredential sp_cred = ServicePrincipalCredentials( client_id=self.client_id, secret=self.client_secret, tenant=self.tenant_id, resource="https://storage.azure.com/", ) token_cred = TokenCredential(sp_cred.token["access_token"]) return token_cred def do_connect(self): """Connect to the BlobServiceClient, using user-specified connection details. Tries credentials first, then connection string and finally account key Raises ------ ValueError if none of the connection details are available """ self.account_url: str = f"https://{self.account_name}.blob.core.windows.net" if self.credential is not None: self.service_client = BlobServiceClient( account_url=self.account_url, credential=self.credential ) elif self.connection_string is not None: self.service_client = BlobServiceClient.from_connection_string( conn_str=self.connection_string ) elif self.account_key is not None: self.service_client = BlobServiceClient( account_url=self.account_url, credential=self.account_key ) else: raise ValueError("unable to connect with provided params!!") def split_path(self, path, delimiter="/", return_container: bool = False, **kwargs): """ Normalize ABFS path string into bucket and key. Parameters ---------- path : string Input path, like `abfs://my_container/path/to/file` delimiter: string Delimiter used to split the path return_container: bool Examples -------- >>> split_path("abfs://my_container/path/to/file") ['my_container', 'path/to/file'] """ if path in ["", delimiter]: return "", "" path = self._strip_protocol(path) path = path.lstrip(delimiter) if "/" not in path: # this means path is the container_name return path, "" else: return path.split(delimiter, 1) # def _generate_blobs(self, *args, **kwargs): # """Follow next_marker to get ALL results.""" # logging.debug("running _generate_blobs...") # blobs = self.blob_fs.list_blobs(*args, **kwargs) # yield from blobs # while blobs.next_marker: # logging.debug(f"following next_marker {blobs.next_marker}") # kwargs["marker"] = blobs.next_marker # blobs = self.blob_fs.list_blobs(*args, **kwargs) # yield from blobs # def _matches( # self, container_name, path, as_directory=False, delimiter="/", **kwargs # ): # """check if the path returns an exact match""" # path = path.rstrip(delimiter) # gen = self.blob_fs.list_blob_names( # container_name=container_name, # prefix=path, # delimiter=delimiter, # num_results=None, # ) # contents = list(gen) # if not contents: # return False # if as_directory: # return contents[0] == path + delimiter # else: # return contents[0] == path def ls( self, path: str, detail: bool = False, invalidate_cache: bool = True, delimiter: str = "/", return_glob: bool = False, **kwargs, ): """ Create a list of blob names from a blob container Parameters ---------- path: str Path to an Azure Blob with its container name detail: bool If False, return a list of blob names, else a list of dictionaries with blob details invalidate_cache: bool If True, do not use the cache delimiter: str Delimiter used to split paths return_glob: bool """ logging.debug(f"abfs.ls() is searching for {path}") container, path = self.split_path(path) if (container in ["", delimiter]) and (path in ["", delimiter]): # This is the case where only the containers are being returned logging.info( "Returning a list of containers in the azure blob storage account" ) if detail: contents = self.service_client.list_containers(include_metadata=True) return self._details(contents) else: contents = self.service_client.list_containers() return [f"{c.name}{delimiter}" for c in contents] else: if container not in ["", delimiter]: # This is the case where the container name is passed container_client = self.service_client.get_container_client( container=container ) blobs = container_client.walk_blobs(name_starts_with=path) try: blobs = [blob for blob in blobs] except Exception: raise FileNotFoundError if len(blobs) > 1: if return_glob: return self._details(blobs, return_glob=True) if detail: return self._details(blobs) else: return [ f"{blob.container}{delimiter}{blob.name}" for blob in blobs ] elif len(blobs) == 1: if (blobs[0].name.rstrip(delimiter) == path) and not blobs[ 0 ].has_key( # NOQA "blob_type" ): path = blobs[0].name blobs = container_client.walk_blobs(name_starts_with=path) if return_glob: return self._details(blobs, return_glob=True) if detail: return self._details(blobs) else: return [ f"{blob.container}{delimiter}{blob.name}" for blob in blobs ] elif isinstance(blobs[0], BlobPrefix): if detail: for blob_page in blobs: return self._details(blob_page) else: outblobs = [] for blob_page in blobs: for blob in blob_page: outblobs.append( f"{blob.container}{delimiter}{blob.name}" ) return outblobs elif blobs[0]["blob_type"] == "BlockBlob": if detail: return self._details(blobs) else: return [ f"{blob.container}{delimiter}{blob.name}" for blob in blobs ] elif isinstance(blobs[0], ItemPaged): outblobs = [] for page in blobs: for b in page: outblobs.append(b) else: raise FileNotFoundError( f"Unable to identify blobs in {path} for {blobs[0].name}" ) elif len(blobs) == 0: if return_glob or (path in ["", delimiter]): return [] else: raise FileNotFoundError else: raise FileNotFoundError def _details(self, contents, delimiter="/", return_glob: bool = False, **kwargs): """ Return a list of dictionaries of specifying details about the contents Parameters ---------- contents delimiter: str Delimiter used to separate containers and files return_glob: bool Returns ------- List of dicts Returns details about the contents, such as name, size and type """ pathlist = [] for c in contents: data = {} if c.has_key("container"): # NOQA data["name"] = f"{c.container}{delimiter}{c.name}" if c.has_key("size"): # NOQA data["size"] = c.size else: data["size"] = 0 if data["size"] == 0: data["type"] = "directory" else: data["type"] = "file" else: data["name"] = f"{c.name}{delimiter}" data["size"] = 0 data["type"] = "directory" if return_glob: data["name"] = data["name"].rstrip("/") pathlist.append(data) return pathlist def walk(self, path: str, maxdepth=None, **kwargs): """ Return all files belows path List all files, recursing into subdirectories; output is iterator-style, like ``os.walk()``. For a simple list of files, ``find()`` is available. Note that the "files" outputted will include anything that is not a directory, such as links. Parameters ---------- path: str Root to recurse into maxdepth: int Maximum recursion depth. None means limitless, but not recommended on link-based file-systems. **kwargs are passed to ``ls`` """ path = self._strip_protocol(path) full_dirs = {} dirs = {} files = {} detail = kwargs.pop("detail", False) try: listing = self.ls(path, detail=True, return_glob=True, **kwargs) except (FileNotFoundError, IOError): return [], [], [] for info in listing: # each info name must be at least [path]/part , but here # we check also for names like [path]/part/ pathname = info["name"].rstrip("/") name = pathname.rsplit("/", 1)[-1] if info["type"] == "directory" and pathname != path: # do not include "self" path full_dirs[pathname] = info dirs[name] = info elif pathname == path: # file-like with same name as give path files[""] = info else: files[name] = info if detail: yield path, dirs, files else: yield path, list(dirs), list(files) if maxdepth is not None: maxdepth -= 1 if maxdepth < 1: return for d in full_dirs: yield from self.walk(d, maxdepth=maxdepth, detail=detail, **kwargs) def mkdir(self, path, delimiter="/", exists_ok=False, **kwargs): """ Create directory entry at path Parameters ---------- path: str The path to create delimiter: str Delimiter to use when splitting the path exists_ok: bool If True, raise an exception if the directory already exists. Defaults to False """ container_name, path = self.split_path(path, delimiter=delimiter) if not exists_ok: if (container_name not in self.ls("")) and (not path): # create new container self.service_client.create_container(name=container_name) elif ( container_name in [container_path.split("/")[0] for container_path in self.ls("")] ) and path: ## attempt to create prefix container_client = self.service_client.get_container_client( container=container_name ) container_client.upload_blob(name=path, data="") else: ## everything else raise RuntimeError(f"Cannot create {container_name}{delimiter}{path}.") else: if container_name in self.ls("") and path: container_client = self.service_client.get_container_client( container=container_name ) container_client.upload_blob(name=path, data="") def rmdir(self, path: str, delimiter="/", **kwargs): """ Remove a directory, if empty Parameters ---------- path: str Path of directory to remove delimiter: str Delimiter to use when splitting the path """ container_name, path = self.split_path(path, delimiter=delimiter) if (container_name + delimiter in self.ls("")) and (not path): # delete container self.service_client.delete_container(container_name) def _rm(self, path, delimiter="/", **kwargs): """ Delete a given file Parameters ---------- path: str Path to file to delete delimiter: str Delimiter to use when splitting the path """ if self.isfile(path): container_name, path = self.split_path(path, delimiter=delimiter) container_client = self.service_client.get_container_client( container=container_name ) logging.debug(f"Delete blob {path} in {container_name}") container_client.delete_blob(path) elif self.isdir(path): container_name, path = self.split_path(path, delimiter=delimiter) container_client = self.service_client.get_container_client( container=container_name ) if (container_name + delimiter in self.ls("")) and (not path): logging.debug(f"Delete container {container_name}") container_client.delete_container(container_name) else: raise RuntimeError(f"cannot delete {path}") def _open( self, path: str, mode: str = "rb", block_size: int = None, autocommit: bool = True, cache_options=None, **kwargs, ): """Open a file on the datalake, or a block blob Parameters ---------- path: str Path to file to open mode: str What mode to open the file in - defaults to "rb" block_size: int Size per block for multi-part downloads. autocommit: bool Whether or not to write to the destination directly cache_type: str One of "readahead", "none", "mmap", "bytes", defaults to "readahead" Caching policy in read mode. See the definitions here: https://filesystem-spec.readthedocs.io/en/latest/api.html#readbuffering """ logging.debug(f"_open: {path}") return AzureBlobFile( fs=self, path=path, mode=mode, block_size=block_size or self.blocksize, autocommit=autocommit, cache_options=cache_options, **kwargs, )
class StorageBlockBlobTest(StorageTestCase): def setUp(self): super(StorageBlockBlobTest, self).setUp() url = self._get_account_url() credential = self._get_shared_key_credential() # test chunking functionality by reducing the size of each chunk, # otherwise the tests would take too long to execute self.bsc = BlobServiceClient(url, credential=credential, connection_data_block_size=4 * 1024, max_single_put_size=32 * 1024, max_block_size=4 * 1024) self.config = self.bsc._config self.container_name = self.get_resource_name('utcontainer') # create source blob to be copied from self.source_blob_name = self.get_resource_name('srcblob') self.source_blob_data = self.get_random_bytes(SOURCE_BLOB_SIZE) blob = self.bsc.get_blob_client(self.container_name, self.source_blob_name) if not self.is_playback(): self.bsc.create_container(self.container_name) blob.upload_blob(self.source_blob_data) # generate a SAS so that it is accessible with a URL sas_token = blob.generate_shared_access_signature( permission=BlobPermissions.READ, expiry=datetime.utcnow() + timedelta(hours=1), ) self.source_blob_url = BlobClient(blob.url, credential=sas_token).url def tearDown(self): if not self.is_playback(): try: self.bsc.delete_container(self.container_name) except HttpResponseError: pass return super(StorageBlockBlobTest, self).tearDown() @record def test_put_block_from_url_and_commit(self): # Arrange dest_blob_name = self.get_resource_name('destblob') dest_blob = self.bsc.get_blob_client(self.container_name, dest_blob_name) # Act part 1: make put block from url calls dest_blob.stage_block_from_url(block_id=1, source_url=self.source_blob_url, source_offset=0, source_length=4 * 1024 - 1) dest_blob.stage_block_from_url(block_id=2, source_url=self.source_blob_url, source_offset=4 * 1024, source_length=8 * 1024) # Assert blocks committed, uncommitted = dest_blob.get_block_list('all') self.assertEqual(len(uncommitted), 2) self.assertEqual(len(committed), 0) # Act part 2: commit the blocks dest_blob.commit_block_list(['1', '2']) # Assert destination blob has right content content = dest_blob.download_blob().content_as_bytes() self.assertEqual(content, self.source_blob_data) @record def test_put_block_from_url_and_validate_content_md5(self): # Arrange dest_blob_name = self.get_resource_name('destblob') dest_blob = self.bsc.get_blob_client(self.container_name, dest_blob_name) src_md5 = StorageContentValidation.get_content_md5( self.source_blob_data) # Act part 1: put block from url with md5 validation dest_blob.stage_block_from_url(block_id=1, source_url=self.source_blob_url, source_content_md5=src_md5, source_offset=0, source_length=8 * 1024) # Assert block was staged committed, uncommitted = dest_blob.get_block_list('all') self.assertEqual(len(uncommitted), 1) self.assertEqual(len(committed), 0) # Act part 2: put block from url with wrong md5 fake_md5 = StorageContentValidation.get_content_md5(b"POTATO") with self.assertRaises(HttpResponseError) as error: dest_blob.stage_block_from_url(block_id=2, source_url=self.source_blob_url, source_content_md5=fake_md5, source_offset=0, source_length=8 * 1024) self.assertEqual(error.exception.error_code, StorageErrorCode.md5_mismatch) # Assert block was not staged committed, uncommitted = dest_blob.get_block_list('all') self.assertEqual(len(uncommitted), 1) self.assertEqual(len(committed), 0) @record def test_copy_blob_sync(self): # Arrange dest_blob_name = self.get_resource_name('destblob') dest_blob = self.bsc.get_blob_client(self.container_name, dest_blob_name) # Act copy_props = dest_blob.copy_blob_from_url(self.source_blob_url, requires_sync=True) # Assert self.assertIsNotNone(copy_props) self.assertIsNotNone(copy_props.copy_id()) self.assertEqual('success', copy_props.status()) # Verify content content = dest_blob.download_blob().content_as_bytes() self.assertEqual(self.source_blob_data, content)
class StorageLargeBlockBlobTest(StorageTestCase): def _setup(self, storage_account, key): # test chunking functionality by reducing the threshold # for chunking and the size of each chunk, otherwise # the tests would take too long to execute self.bsc = BlobServiceClient(self.account_url(storage_account, "blob"), credential=key, max_single_put_size=32 * 1024, max_block_size=2 * 1024 * 1024, min_large_block_upload_threshold=1 * 1024 * 1024) self.config = self.bsc._config self.container_name = self.get_resource_name('utcontainer') if self.is_live: self.bsc.create_container(self.container_name) def _teardown(self, file_name): if path.isfile(file_name): try: remove(file_name) except: pass # --Helpers----------------------------------------------------------------- def _get_blob_reference(self): return self.get_resource_name(TEST_BLOB_PREFIX) def _create_blob(self): blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) blob.upload_blob(b'') return blob def assertBlobEqual(self, container_name, blob_name, expected_data): blob = self.bsc.get_blob_client(container_name, blob_name) actual_data = blob.download_blob() self.assertEqual(b"".join(list(actual_data.chunks())), expected_data) # --Test cases for block blobs -------------------------------------------- @pytest.mark.live_test_only @GlobalStorageAccountPreparer() def test_put_block_bytes_large(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) blob = self._create_blob() # Act for i in range(5): resp = blob.stage_block('block {0}'.format(i).encode('utf-8'), urandom(LARGE_BLOCK_SIZE)) self.assertIsNotNone(resp) assert 'content_md5' in resp assert 'content_crc64' in resp assert 'request_id' in resp # Assert @pytest.mark.live_test_only @GlobalStorageAccountPreparer() def test_put_block_bytes_large_with_md5(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) blob = self._create_blob() # Act for i in range(5): resp = blob.stage_block('block {0}'.format(i).encode('utf-8'), urandom(LARGE_BLOCK_SIZE), validate_content=True) self.assertIsNotNone(resp) assert 'content_md5' in resp assert 'content_crc64' in resp assert 'request_id' in resp @pytest.mark.live_test_only @GlobalStorageAccountPreparer() def test_put_block_stream_large(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) blob = self._create_blob() # Act for i in range(5): stream = BytesIO(bytearray(LARGE_BLOCK_SIZE)) resp = resp = blob.stage_block( 'block {0}'.format(i).encode('utf-8'), stream, length=LARGE_BLOCK_SIZE) self.assertIsNotNone(resp) assert 'content_md5' in resp assert 'content_crc64' in resp assert 'request_id' in resp # Assert @pytest.mark.live_test_only @GlobalStorageAccountPreparer() def test_put_block_stream_large_with_md5(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) blob = self._create_blob() # Act for i in range(5): stream = BytesIO(bytearray(LARGE_BLOCK_SIZE)) resp = resp = blob.stage_block( 'block {0}'.format(i).encode('utf-8'), stream, length=LARGE_BLOCK_SIZE, validate_content=True) self.assertIsNotNone(resp) assert 'content_md5' in resp assert 'content_crc64' in resp assert 'request_id' in resp # Assert @pytest.mark.live_test_only @GlobalStorageAccountPreparer() def test_create_large_blob_from_path(self, resource_group, location, storage_account, storage_account_key): # parallel tests introduce random order of requests, can only run live self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = bytearray(urandom(LARGE_BLOB_SIZE)) FILE_PATH = 'large_blob_from_path.temp.{}.dat'.format(str( uuid.uuid4())) with open(FILE_PATH, 'wb') as stream: stream.write(data) # Act with open(FILE_PATH, 'rb') as stream: blob.upload_blob(stream, max_concurrency=2) # Assert self.assertBlobEqual(self.container_name, blob_name, data) self._teardown(FILE_PATH) @pytest.mark.live_test_only @GlobalStorageAccountPreparer() def test_create_large_blob_from_path_with_md5(self, resource_group, location, storage_account, storage_account_key): # parallel tests introduce random order of requests, can only run live self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = bytearray(urandom(LARGE_BLOB_SIZE)) FILE_PATH = "blob_from_path_with_md5.temp.dat" with open(FILE_PATH, 'wb') as stream: stream.write(data) # Act with open(FILE_PATH, 'rb') as stream: blob.upload_blob(stream, validate_content=True, max_concurrency=2) # Assert self.assertBlobEqual(self.container_name, blob_name, data) self._teardown(FILE_PATH) @pytest.mark.live_test_only @GlobalStorageAccountPreparer() def test_create_large_blob_from_path_non_parallel(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = bytearray(self.get_random_bytes(100)) FILE_PATH = "blob_from_path_non_parallel.temp.dat" with open(FILE_PATH, 'wb') as stream: stream.write(data) # Act with open(FILE_PATH, 'rb') as stream: blob.upload_blob(stream, max_concurrency=1) # Assert self.assertBlobEqual(self.container_name, blob_name, data) self._teardown(FILE_PATH) @pytest.mark.live_test_only @GlobalStorageAccountPreparer() def test_create_large_blob_from_path_with_progress(self, resource_group, location, storage_account, storage_account_key): # parallel tests introduce random order of requests, can only run live self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = bytearray(urandom(LARGE_BLOB_SIZE)) FILE_PATH = "blob_from_path_with_progress.temp.dat" with open(FILE_PATH, 'wb') as stream: stream.write(data) # Act progress = [] def callback(response): current = response.context['upload_stream_current'] total = response.context['data_stream_total'] if current is not None: progress.append((current, total)) with open(FILE_PATH, 'rb') as stream: blob.upload_blob(stream, max_concurrency=2, raw_response_hook=callback) # Assert self.assertBlobEqual(self.container_name, blob_name, data) self.assert_upload_progress(len(data), self.config.max_block_size, progress) self._teardown(FILE_PATH) @pytest.mark.live_test_only @GlobalStorageAccountPreparer() def test_create_large_blob_from_path_with_properties( self, resource_group, location, storage_account, storage_account_key): # parallel tests introduce random order of requests, can only run live self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = bytearray(urandom(LARGE_BLOB_SIZE)) FILE_PATH = 'blob_from_path_with_properties.temp.{}.dat'.format( str(uuid.uuid4())) with open(FILE_PATH, 'wb') as stream: stream.write(data) # Act content_settings = ContentSettings(content_type='image/png', content_language='spanish') with open(FILE_PATH, 'rb') as stream: blob.upload_blob(stream, content_settings=content_settings, max_concurrency=2) # Assert self.assertBlobEqual(self.container_name, blob_name, data) properties = blob.get_blob_properties() self.assertEqual(properties.content_settings.content_type, content_settings.content_type) self.assertEqual(properties.content_settings.content_language, content_settings.content_language) self._teardown(FILE_PATH) @pytest.mark.live_test_only @GlobalStorageAccountPreparer() def test_create_large_blob_from_stream_chunked_upload( self, resource_group, location, storage_account, storage_account_key): # parallel tests introduce random order of requests, can only run live self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = bytearray(urandom(LARGE_BLOB_SIZE)) FILE_PATH = 'blob_from_stream_chunked_upload.temp.{}.dat'.format( str(uuid.uuid4())) with open(FILE_PATH, 'wb') as stream: stream.write(data) # Act with open(FILE_PATH, 'rb') as stream: blob.upload_blob(stream, max_concurrency=2) # Assert self.assertBlobEqual(self.container_name, blob_name, data) self._teardown(FILE_PATH) @pytest.mark.live_test_only @GlobalStorageAccountPreparer() def test_creat_lrgblob_frm_stream_w_progress_chnkd_upload( self, resource_group, location, storage_account, storage_account_key): # parallel tests introduce random order of requests, can only run live self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = bytearray(urandom(LARGE_BLOB_SIZE)) FILE_PATH = 'stream_w_progress_chnkd_upload.temp.{}.dat'.format( str(uuid.uuid4())) with open(FILE_PATH, 'wb') as stream: stream.write(data) # Act progress = [] def callback(response): current = response.context['upload_stream_current'] total = response.context['data_stream_total'] if current is not None: progress.append((current, total)) with open(FILE_PATH, 'rb') as stream: blob.upload_blob(stream, max_concurrency=2, raw_response_hook=callback) # Assert self.assertBlobEqual(self.container_name, blob_name, data) self.assert_upload_progress(len(data), self.config.max_block_size, progress) self._teardown(FILE_PATH) @pytest.mark.live_test_only @GlobalStorageAccountPreparer() def test_create_large_blob_from_stream_chunked_upload_with_count( self, resource_group, location, storage_account, storage_account_key): # parallel tests introduce random order of requests, can only run live self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = bytearray(urandom(LARGE_BLOB_SIZE)) FILE_PATH = 'chunked_upload_with_count.temp.{}.dat'.format( str(uuid.uuid4())) with open(FILE_PATH, 'wb') as stream: stream.write(data) # Act blob_size = len(data) - 301 with open(FILE_PATH, 'rb') as stream: blob.upload_blob(stream, length=blob_size, max_concurrency=2) # Assert self.assertBlobEqual(self.container_name, blob_name, data[:blob_size]) self._teardown(FILE_PATH) @pytest.mark.live_test_only @GlobalStorageAccountPreparer() def test_creat_lrgblob_frm_strm_chnkd_uplod_w_count_n_props( self, resource_group, location, storage_account, storage_account_key): # parallel tests introduce random order of requests, can only run live self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = bytearray(urandom(LARGE_BLOB_SIZE)) FILE_PATH = 'plod_w_count_n_props.temp.{}.dat'.format(str( uuid.uuid4())) with open(FILE_PATH, 'wb') as stream: stream.write(data) # Act content_settings = ContentSettings(content_type='image/png', content_language='spanish') blob_size = len(data) - 301 with open(FILE_PATH, 'rb') as stream: blob.upload_blob(stream, length=blob_size, content_settings=content_settings, max_concurrency=2) # Assert self.assertBlobEqual(self.container_name, blob_name, data[:blob_size]) properties = blob.get_blob_properties() self.assertEqual(properties.content_settings.content_type, content_settings.content_type) self.assertEqual(properties.content_settings.content_language, content_settings.content_language) self._teardown(FILE_PATH) @pytest.mark.live_test_only @GlobalStorageAccountPreparer() def test_creat_lrg_blob_frm_stream_chnked_upload_w_props( self, resource_group, location, storage_account, storage_account_key): # parallel tests introduce random order of requests, can only run live self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = bytearray(urandom(LARGE_BLOB_SIZE)) FILE_PATH = 'creat_lrg_blob.temp.{}.dat'.format(str(uuid.uuid4())) with open(FILE_PATH, 'wb') as stream: stream.write(data) # Act content_settings = ContentSettings(content_type='image/png', content_language='spanish') with open(FILE_PATH, 'rb') as stream: blob.upload_blob(stream, content_settings=content_settings, max_concurrency=2) # Assert self.assertBlobEqual(self.container_name, blob_name, data) properties = blob.get_blob_properties() self.assertEqual(properties.content_settings.content_type, content_settings.content_type) self.assertEqual(properties.content_settings.content_language, content_settings.content_language) self._teardown(FILE_PATH)
class StorageBlobTagsTest(StorageTestCase): def _setup(self, storage_account, key): self.bsc = BlobServiceClient(self.account_url(storage_account, "blob"), credential=key) self.container_name = self.get_resource_name("container") if self.is_live: container = self.bsc.get_container_client(self.container_name) try: container.create_container(timeout=5) except ResourceExistsError: pass self.byte_data = self.get_random_bytes(1024) def _teardown(self, FILE_PATH): if os.path.isfile(FILE_PATH): try: os.remove(FILE_PATH) except: pass #--Helpers----------------------------------------------------------------- def _get_blob_reference(self): return self.get_resource_name(TEST_BLOB_PREFIX) def _create_block_blob(self, tags=None, container_name=None, blob_name=None): blob_name = blob_name or self._get_blob_reference() blob_client = self.bsc.get_blob_client(container_name or self.container_name, blob_name) resp = blob_client.upload_blob(self.byte_data, length=len(self.byte_data), overwrite=True, tags=tags) return blob_client, resp def _create_empty_block_blob(self, tags=None): blob_name = self._get_blob_reference() blob_client = self.bsc.get_blob_client(self.container_name, blob_name) resp = blob_client.upload_blob(b'', length=0, overwrite=True, tags=tags) return blob_client, resp def _create_append_blob(self, tags=None): blob_name = self._get_blob_reference() blob_client = self.bsc.get_blob_client(self.container_name, blob_name) resp = blob_client.create_append_blob(tags=tags) return blob_client, resp def _create_page_blob(self, tags=None): blob_name = self._get_blob_reference() blob_client = self.bsc.get_blob_client(self.container_name, blob_name) resp = blob_client.create_page_blob(tags=tags, size=512) return blob_client, resp def _create_container(self, prefix="container"): container_name = self.get_resource_name(prefix) try: self.bsc.create_container(container_name) except: pass return container_name #-- test cases for blob tags ---------------------------------------------- @GlobalResourceGroupPreparer() @StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage') def test_set_blob_tags(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) blob_client, _ = self._create_block_blob() # Act blob_tags = {"tag1": "firsttag", "tag2": "secondtag", "tag3": "thirdtag"} resp = blob_client.set_blob_tags(blob_tags) # Assert self.assertIsNotNone(resp) @pytest.mark.playback_test_only @GlobalStorageAccountPreparer() def test_set_blob_tags_for_a_version(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) # use this version to set tag blob_client, resp = self._create_block_blob() self._create_block_blob() # TODO: enable versionid for this account and test set tag for a version # Act tags = {"tag1": "firsttag", "tag2": "secondtag", "tag3": "thirdtag"} resp = blob_client.set_blob_tags(tags, version_id=resp['version_id']) # Assert self.assertIsNotNone(resp) @GlobalResourceGroupPreparer() @StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage') def test_get_blob_tags(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) blob_client, resp = self._create_block_blob() # Act tags = {"tag1": "firsttag", "tag2": "secondtag", "tag3": "thirdtag"} blob_client.set_blob_tags(tags) resp = blob_client.get_blob_tags() # Assert self.assertIsNotNone(resp) self.assertEqual(len(resp), 3) for key, value in resp.items(): self.assertEqual(tags[key], value) @GlobalResourceGroupPreparer() @StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage') def test_get_blob_tags_for_a_snapshot(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) tags = {"+-./:=_ ": "firsttag", "tag2": "+-./:=_", "+-./:=_1": "+-./:=_"} blob_client, resp = self._create_block_blob(tags=tags) snapshot = blob_client.create_snapshot() snapshot_client = self.bsc.get_blob_client(self.container_name, blob_client.blob_name, snapshot=snapshot) resp = snapshot_client.get_blob_tags() # Assert self.assertIsNotNone(resp) self.assertEqual(len(resp), 3) for key, value in resp.items(): self.assertEqual(tags[key], value) @GlobalResourceGroupPreparer() @StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage') def test_upload_block_blob_with_tags(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) tags = {"tag1": "firsttag", "tag2": "secondtag", "tag3": "thirdtag"} blob_client, resp = self._create_block_blob(tags=tags) resp = blob_client.get_blob_tags() # Assert self.assertIsNotNone(resp) self.assertEqual(len(resp), 3) @GlobalResourceGroupPreparer() @StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage') def test_get_blob_properties_returns_tags_num(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) tags = {"tag1": "firsttag", "tag2": "secondtag", "tag3": "thirdtag"} blob_client, resp = self._create_block_blob(tags=tags) resp = blob_client.get_blob_properties() downloaded = blob_client.download_blob() # Assert self.assertIsNotNone(resp) self.assertEqual(resp.tag_count, len(tags)) self.assertEqual(downloaded.properties.tag_count, len(tags)) @GlobalResourceGroupPreparer() @StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage') def test_create_append_blob_with_tags(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) tags = {"+-./:=_ ": "firsttag", "tag2": "+-./:=_", "+-./:=_1": "+-./:=_"} blob_client, resp = self._create_append_blob(tags=tags) resp = blob_client.get_blob_tags() # Assert self.assertIsNotNone(resp) self.assertEqual(len(resp), 3) @GlobalResourceGroupPreparer() @StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage') def test_create_page_blob_with_tags(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) tags = {"tag1": "firsttag", "tag2": "secondtag", "tag3": "thirdtag"} blob_client, resp = self._create_page_blob(tags=tags) resp = blob_client.get_blob_tags() # Assert self.assertIsNotNone(resp) self.assertEqual(len(resp), 3) @GlobalResourceGroupPreparer() @StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage') def test_commit_block_list_with_tags(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) tags = {"tag1": "firsttag", "tag2": "secondtag", "tag3": "thirdtag"} blob_client, resp = self._create_empty_block_blob(tags={'condition tag': 'test tag'}) blob_client.stage_block('1', b'AAA') blob_client.stage_block('2', b'BBB') blob_client.stage_block('3', b'CCC') # Act block_list = [BlobBlock(block_id='1'), BlobBlock(block_id='2'), BlobBlock(block_id='3')] with self.assertRaises(ResourceModifiedError): blob_client.commit_block_list(block_list, tags=tags, if_tags_match_condition="\"condition tag\"='wrong tag'") blob_client.commit_block_list(block_list, tags=tags, if_tags_match_condition="\"condition tag\"='test tag'") resp = blob_client.get_blob_tags() # Assert self.assertIsNotNone(resp) self.assertEqual(len(resp), len(tags)) @GlobalResourceGroupPreparer() @StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage') def test_start_copy_from_url_with_tags(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) tags = {"tag1": "firsttag", "tag2": "secondtag", "tag3": "thirdtag"} blob_client, resp = self._create_block_blob() # Act sourceblob = '{0}/{1}/{2}'.format( self.account_url(storage_account, "blob"), self.container_name, blob_client.blob_name) copyblob = self.bsc.get_blob_client(self.container_name, 'blob1copy') copy = copyblob.start_copy_from_url(sourceblob, tags=tags) # Assert self.assertIsNotNone(copy) self.assertEqual(copy['copy_status'], 'success') self.assertFalse(isinstance(copy['copy_status'], Enum)) self.assertIsNotNone(copy['copy_id']) copy_content = copyblob.download_blob().readall() self.assertEqual(copy_content, self.byte_data) resp = copyblob.get_blob_tags() # Assert self.assertIsNotNone(resp) self.assertEqual(len(resp), len(tags)) @GlobalResourceGroupPreparer() @StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage') def test_list_blobs_returns_tags(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) tags = {"tag1": "firsttag", "tag2": "secondtag", "tag3": "thirdtag"} self._create_block_blob(tags=tags) container = self.bsc.get_container_client(self.container_name) blob_list = container.list_blobs(include="tags") #Assert for blob in blob_list: self.assertEqual(blob.tag_count, len(tags)) for key, value in blob.tags.items(): self.assertEqual(tags[key], value) @GlobalResourceGroupPreparer() @StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage') def test_filter_blobs(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) container_name1 = self._create_container(prefix="container1") container_name2 = self._create_container(prefix="container2") container_name3 = self._create_container(prefix="container3") tags = {"tag1": "firsttag", "tag2": "secondtag", "tag3": "thirdtag"} self._create_block_blob(tags=tags, blob_name="blob1") self._create_block_blob(tags=tags, blob_name="blob2", container_name=container_name1) self._create_block_blob(tags=tags, blob_name="blob3", container_name=container_name2) self._create_block_blob(tags=tags, blob_name="blob4", container_name=container_name3) if self.is_live: sleep(10) where = "tag1='firsttag'" blob_list = self.bsc.find_blobs_by_tags(filter_expression=where, results_per_page=2).by_page() first_page = next(blob_list) items_on_page1 = list(first_page) second_page = next(blob_list) items_on_page2 = list(second_page) self.assertEqual(2, len(items_on_page1)) self.assertEqual(2, len(items_on_page2)) @pytest.mark.live_test_only @GlobalResourceGroupPreparer() @StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage') def test_filter_blobs_using_account_sas(self, resource_group, location, storage_account, storage_account_key): token = generate_account_sas( storage_account.name, storage_account_key, ResourceTypes(service=True, container=True, object=True), AccountSasPermissions(write=True, list=True, read=True, delete_previous_version=True, tag=True, filter_by_tags=True), datetime.utcnow() + timedelta(hours=1), ) self._setup(storage_account, token) tags = {"year": '1000', "tag2": "secondtag", "tag3": "thirdtag", "habitat_type": 'Shallow Lowland Billabongs'} blob_client, _ = self._create_block_blob(tags=tags, container_name=self.container_name) blob_client.set_blob_tags(tags=tags) tags_on_blob = blob_client.get_blob_tags() self.assertEqual(len(tags_on_blob), len(tags)) if self.is_live: sleep(10) # To filter in a specific container use: # where = "@container='{}' and tag1='1000' and tag2 = 'secondtag'".format(container_name1) where = "\"year\"='1000' and tag2 = 'secondtag' and tag3='thirdtag'" blob_list = self.bsc.find_blobs_by_tags(filter_expression=where, results_per_page=2).by_page() first_page = next(blob_list) items_on_page1 = list(first_page) self.assertEqual(1, len(items_on_page1)) @pytest.mark.live_test_only @GlobalResourceGroupPreparer() @StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage') def test_set_blob_tags_using_blob_sas(self, resource_group, location, storage_account, storage_account_key): token = generate_account_sas( storage_account.name, storage_account_key, ResourceTypes(service=True, container=True, object=True), AccountSasPermissions(write=True, list=True, read=True, delete_previous_version=True, tag=True, filter_by_tags=True), datetime.utcnow() + timedelta(hours=1), ) self._setup(storage_account, token) tags = {"year": '1000', "tag2": "secondtag", "tag3": "thirdtag", "habitat_type": 'Shallow Lowland Billabongs'} blob_client, _ = self._create_block_blob(tags=tags, container_name=self.container_name) token1 = generate_blob_sas( storage_account.name, self.container_name, blob_client.blob_name, account_key=storage_account_key, permission=BlobSasPermissions(delete_previous_version=True, tag=True), expiry=datetime.utcnow() + timedelta(hours=1), ) blob_client=BlobClient.from_blob_url(blob_client.url, token1) blob_client.set_blob_tags(tags=tags) tags_on_blob = blob_client.get_blob_tags() self.assertEqual(len(tags_on_blob), len(tags)) if self.is_live: sleep(10) # To filter in a specific container use: # where = "@container='{}' and tag1='1000' and tag2 = 'secondtag'".format(container_name1) where = "\"year\"='1000' and tag2 = 'secondtag' and tag3='thirdtag'" blob_list = self.bsc.find_blobs_by_tags(filter_expression=where, results_per_page=2).by_page() first_page = next(blob_list) items_on_page1 = list(first_page) self.assertEqual(1, len(items_on_page1))
if delete_extra_containers: # c = extra_containers[0] for c in extra_containers: print('Delete container {} from storage account {}?'.format( c, target_blob_service_client.account_name)) if confirm(): target_blob_service_client.delete_container(c) #%% Create missing containers # c = missing_containers[0] for c in missing_containers: print('Creating container {} in storage account {}'.format( c, target_blob_service_client.account_name)) target_blob_service_client.create_container(c) #%% Generate azcopy commands azcopy_commands = [] # c = source_container_names[0] for c in source_container_names: cmd = 'azcopy sync "' cmd += source_account_url_blob + '/' + c + '/' + source_sas_token cmd += '" "' cmd += target_account_url_blob + '/' + c + '/' + target_sas_token cmd += '" --delete-destination=True --log-level=NONE' azcopy_commands.append(cmd)
class StorageBlockBlobTest(StorageTestCase): def _setup(self, storage_account, key): self.bsc = BlobServiceClient(self.account_url(storage_account, "blob"), credential=key, connection_data_block_size=4 * 1024, max_single_put_size=32 * 1024, max_block_size=4 * 1024) self.config = self.bsc._config self.container_name = self.get_resource_name('utcontainer') # create source blob to be copied from self.source_blob_name = self.get_resource_name('srcblob') self.source_blob_data = self.get_random_bytes(SOURCE_BLOB_SIZE) blob = self.bsc.get_blob_client(self.container_name, self.source_blob_name) if self.is_live: self.bsc.create_container(self.container_name) blob.upload_blob(self.source_blob_data) # generate a SAS so that it is accessible with a URL sas_token = generate_blob_sas( blob.account_name, blob.container_name, blob.blob_name, snapshot=blob.snapshot, account_key=blob.credential.account_key, permission=BlobSasPermissions(read=True), expiry=datetime.utcnow() + timedelta(hours=1), ) self.source_blob_url = BlobClient.from_blob_url( blob.url, credential=sas_token).url @GlobalStorageAccountPreparer() def test_put_block_from_url_and_commit(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) dest_blob_name = self.get_resource_name('destblob') dest_blob = self.bsc.get_blob_client(self.container_name, dest_blob_name) # Act part 1: make put block from url calls split = 4 * 1024 dest_blob.stage_block_from_url(block_id=1, source_url=self.source_blob_url, source_offset=0, source_length=split) dest_blob.stage_block_from_url(block_id=2, source_url=self.source_blob_url, source_offset=split, source_length=split) # Assert blocks committed, uncommitted = dest_blob.get_block_list('all') self.assertEqual(len(uncommitted), 2) self.assertEqual(len(committed), 0) # Act part 2: commit the blocks dest_blob.commit_block_list(['1', '2']) # Assert destination blob has right content content = dest_blob.download_blob().readall() self.assertEqual(len(content), 8 * 1024) self.assertEqual(content, self.source_blob_data) @GlobalStorageAccountPreparer() def test_put_block_from_url_and_validate_content_md5( self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) dest_blob_name = self.get_resource_name('destblob') dest_blob = self.bsc.get_blob_client(self.container_name, dest_blob_name) src_md5 = StorageContentValidation.get_content_md5( self.source_blob_data) # Act part 1: put block from url with md5 validation dest_blob.stage_block_from_url(block_id=1, source_url=self.source_blob_url, source_content_md5=src_md5, source_offset=0, source_length=8 * 1024) # Assert block was staged committed, uncommitted = dest_blob.get_block_list('all') self.assertEqual(len(uncommitted), 1) self.assertEqual(len(committed), 0) # Act part 2: put block from url with wrong md5 fake_md5 = StorageContentValidation.get_content_md5(b"POTATO") with self.assertRaises(HttpResponseError) as error: dest_blob.stage_block_from_url(block_id=2, source_url=self.source_blob_url, source_content_md5=fake_md5, source_offset=0, source_length=8 * 1024) self.assertEqual(error.exception.error_code, StorageErrorCode.md5_mismatch) # Assert block was not staged committed, uncommitted = dest_blob.get_block_list('all') self.assertEqual(len(uncommitted), 1) self.assertEqual(len(committed), 0) @GlobalStorageAccountPreparer() def test_copy_blob_sync(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) dest_blob_name = self.get_resource_name('destblob') dest_blob = self.bsc.get_blob_client(self.container_name, dest_blob_name) # Act copy_props = dest_blob.start_copy_from_url(self.source_blob_url, requires_sync=True) # Assert self.assertIsNotNone(copy_props) self.assertIsNotNone(copy_props['copy_id']) self.assertEqual('success', copy_props['copy_status']) # Verify content content = dest_blob.download_blob().readall() self.assertEqual(self.source_blob_data, content) @pytest.mark.playback_test_only @GlobalStorageAccountPreparer() def test_sync_copy_blob_returns_vid(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) dest_blob_name = self.get_resource_name('destblob') dest_blob = self.bsc.get_blob_client(self.container_name, dest_blob_name) # Act copy_props = dest_blob.start_copy_from_url(self.source_blob_url, requires_sync=True) # Assert self.assertIsNotNone(copy_props['version_id']) self.assertIsNotNone(copy_props) self.assertIsNotNone(copy_props['copy_id']) self.assertEqual('success', copy_props['copy_status']) # Verify content content = dest_blob.download_blob().readall() self.assertEqual(self.source_blob_data, content)
class StorageBlockBlobTest(StorageTestCase): def setUp(self): super(StorageBlockBlobTest, self).setUp() url = self._get_account_url() # test chunking functionality by reducing the size of each chunk, # otherwise the tests would take too long to execute self.bsc = BlobServiceClient( url, credential=self.settings.STORAGE_ACCOUNT_KEY, connection_data_block_size=4 * 1024, max_single_put_size=32 * 1024, max_block_size=4 * 1024) self.config = self.bsc._config self.container_name = self.get_resource_name('utcontainer') if not self.is_playback(): self.bsc.create_container(self.container_name) def tearDown(self): if not self.is_playback(): try: self.bsc.delete_container(self.container_name) except: pass if os.path.isfile(FILE_PATH): try: os.remove(FILE_PATH) except: pass return super(StorageBlockBlobTest, self).tearDown() #--Helpers----------------------------------------------------------------- def _get_blob_reference(self): return self.get_resource_name(TEST_BLOB_PREFIX) def _create_blob(self): blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) blob.upload_blob(b'') return blob def assertBlobEqual(self, container_name, blob_name, expected_data): blob = self.bsc.get_blob_client(container_name, blob_name) actual_data = blob.download_blob() self.assertEqual(b"".join(list(actual_data)), expected_data) class NonSeekableFile(object): def __init__(self, wrapped_file): self.wrapped_file = wrapped_file def write(self, data): self.wrapped_file.write(data) def read(self, count): return self.wrapped_file.read(count) #--Test cases for block blobs -------------------------------------------- @record def test_put_block(self): # Arrange blob = self._create_blob() # Act for i in range(5): resp = blob.stage_block(i, 'block {0}'.format(i).encode('utf-8')) self.assertIsNone(resp) # Assert @record def test_put_block_unicode(self): # Arrange blob = self._create_blob() # Act resp = blob.stage_block('1', u'啊齄丂狛狜') self.assertIsNone(resp) # Assert @record def test_put_block_with_md5(self): # Arrange blob = self._create_blob() # Act blob.stage_block(1, b'block', validate_content=True) # Assert @record def test_put_block_list(self): # Arrange blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) blob.stage_block('1', b'AAA') blob.stage_block('2', b'BBB') blob.stage_block('3', b'CCC') # Act block_list = [ BlobBlock(block_id='1'), BlobBlock(block_id='2'), BlobBlock(block_id='3') ] put_block_list_resp = blob.commit_block_list(block_list) # Assert content = blob.download_blob() self.assertEqual(b"".join(list(content)), b'AAABBBCCC') self.assertEqual(content.properties.etag, put_block_list_resp.get('etag')) self.assertEqual(content.properties.last_modified, put_block_list_resp.get('last_modified')) @record def test_put_block_list_invalid_block_id(self): # Arrange blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) blob.stage_block('1', b'AAA') blob.stage_block('2', b'BBB') blob.stage_block('3', b'CCC') # Act try: block_list = [ BlobBlock(block_id='1'), BlobBlock(block_id='2'), BlobBlock(block_id='4') ] blob.commit_block_list(block_list) self.fail() except HttpResponseError as e: self.assertGreaterEqual( str(e).find('specified block list is invalid'), 0) # Assert @record def test_put_block_list_with_md5(self): # Arrange blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) blob.stage_block('1', b'AAA') blob.stage_block('2', b'BBB') blob.stage_block('3', b'CCC') # Act block_list = [ BlobBlock(block_id='1'), BlobBlock(block_id='2'), BlobBlock(block_id='3') ] blob.commit_block_list(block_list, validate_content=True) # Assert @record def test_get_block_list_no_blocks(self): # Arrange blob = self._create_blob() # Act block_list = blob.get_block_list('all') # Assert self.assertIsNotNone(block_list) self.assertEqual(len(block_list[1]), 0) self.assertEqual(len(block_list[0]), 0) @record def test_get_block_list_uncommitted_blocks(self): # Arrange blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) blob.stage_block('1', b'AAA') blob.stage_block('2', b'BBB') blob.stage_block('3', b'CCC') # Act block_list = blob.get_block_list('uncommitted') # Assert self.assertIsNotNone(block_list) self.assertEqual(len(block_list), 2) self.assertEqual(len(block_list[1]), 3) self.assertEqual(len(block_list[0]), 0) self.assertEqual(block_list[1][0].id, '1') self.assertEqual(block_list[1][0].size, 3) self.assertEqual(block_list[1][1].id, '2') self.assertEqual(block_list[1][1].size, 3) self.assertEqual(block_list[1][2].id, '3') self.assertEqual(block_list[1][2].size, 3) @record def test_get_block_list_committed_blocks(self): # Arrange blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) blob.stage_block('1', b'AAA') blob.stage_block('2', b'BBB') blob.stage_block('3', b'CCC') block_list = [ BlobBlock(block_id='1'), BlobBlock(block_id='2'), BlobBlock(block_id='3') ] blob.commit_block_list(block_list) # Act block_list = blob.get_block_list('committed') # Assert self.assertIsNotNone(block_list) self.assertEqual(len(block_list), 2) self.assertEqual(len(block_list[1]), 0) self.assertEqual(len(block_list[0]), 3) self.assertEqual(block_list[0][0].id, '1') self.assertEqual(block_list[0][0].size, 3) self.assertEqual(block_list[0][1].id, '2') self.assertEqual(block_list[0][1].size, 3) self.assertEqual(block_list[0][2].id, '3') self.assertEqual(block_list[0][2].size, 3) @record def test_create_small_block_blob_with_no_overwrite(self): # Arrange blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data1 = b'hello world' data2 = b'hello second world' # Act create_resp = blob.upload_blob(data1, overwrite=True) with self.assertRaises(ResourceExistsError): blob.upload_blob(data2, overwrite=False) props = blob.get_blob_properties() # Assert self.assertBlobEqual(self.container_name, blob_name, data1) self.assertEqual(props.etag, create_resp.get('etag')) self.assertEqual(props.last_modified, create_resp.get('last_modified')) self.assertEqual(props.blob_type, BlobType.BlockBlob) @record def test_create_small_block_blob_with_overwrite(self): # Arrange blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data1 = b'hello world' data2 = b'hello second world' # Act create_resp = blob.upload_blob(data1, overwrite=True) update_resp = blob.upload_blob(data2, overwrite=True) props = blob.get_blob_properties() # Assert self.assertBlobEqual(self.container_name, blob_name, data2) self.assertEqual(props.etag, update_resp.get('etag')) self.assertEqual(props.last_modified, update_resp.get('last_modified')) self.assertEqual(props.blob_type, BlobType.BlockBlob) @record def test_create_large_block_blob_with_no_overwrite(self): # Arrange blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data1 = self.get_random_bytes(LARGE_BLOB_SIZE) data2 = self.get_random_bytes(LARGE_BLOB_SIZE) # Act create_resp = blob.upload_blob(data1, overwrite=True, metadata={'BlobData': 'Data1'}) with self.assertRaises(ResourceExistsError): blob.upload_blob(data2, overwrite=False, metadata={'BlobData': 'Data2'}) props = blob.get_blob_properties() # Assert self.assertBlobEqual(self.container_name, blob_name, data1) self.assertEqual(props.etag, create_resp.get('etag')) self.assertEqual(props.last_modified, create_resp.get('last_modified')) self.assertEqual(props.blob_type, BlobType.BlockBlob) self.assertEqual(props.metadata, {'BlobData': 'Data1'}) self.assertEqual(props.size, LARGE_BLOB_SIZE) @record def test_create_large_block_blob_with_overwrite(self): # Arrange blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data1 = self.get_random_bytes(LARGE_BLOB_SIZE) data2 = self.get_random_bytes(LARGE_BLOB_SIZE + 512) # Act create_resp = blob.upload_blob(data1, overwrite=True, metadata={'BlobData': 'Data1'}) update_resp = blob.upload_blob(data2, overwrite=True, metadata={'BlobData': 'Data2'}) props = blob.get_blob_properties() # Assert self.assertBlobEqual(self.container_name, blob_name, data2) self.assertEqual(props.etag, update_resp.get('etag')) self.assertEqual(props.last_modified, update_resp.get('last_modified')) self.assertEqual(props.blob_type, BlobType.BlockBlob) self.assertEqual(props.metadata, {'BlobData': 'Data2'}) self.assertEqual(props.size, LARGE_BLOB_SIZE + 512) @record def test_create_blob_from_bytes_single_put(self): # Arrange blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = b'hello world' # Act create_resp = blob.upload_blob(data) props = blob.get_blob_properties() # Assert self.assertBlobEqual(self.container_name, blob_name, data) self.assertEqual(props.etag, create_resp.get('etag')) self.assertEqual(props.last_modified, create_resp.get('last_modified')) @record def test_create_blob_from_0_bytes(self): # Arrange blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = b'' # Act create_resp = blob.upload_blob(data) props = blob.get_blob_properties() # Assert self.assertBlobEqual(self.container_name, blob_name, data) self.assertEqual(props.etag, create_resp.get('etag')) self.assertEqual(props.last_modified, create_resp.get('last_modified')) @record def test_create_from_bytes_blob_unicode(self): # Arrange blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = u'hello world' # Act create_resp = blob.upload_blob(data) props = blob.get_blob_properties() # Assert self.assertBlobEqual(self.container_name, blob_name, data) self.assertEqual(props.etag, create_resp.get('etag')) self.assertEqual(props.last_modified, create_resp.get('last_modified')) @record def test_create_from_bytes_blob_unicode(self): # Arrange blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) # Act data = u'hello world' create_resp = blob.upload_blob(data) props = blob.get_blob_properties() # Assert self.assertBlobEqual(self.container_name, blob_name, data.encode('utf-8')) self.assertEqual(props.etag, create_resp.get('etag')) self.assertEqual(props.last_modified, create_resp.get('last_modified')) def test_create_from_bytes_blob_with_lease_id(self): # parallel tests introduce random order of requests, can only run live if TestMode.need_recording_file(self.test_mode): return # Arrange blob = self._create_blob() data = self.get_random_bytes(LARGE_BLOB_SIZE) lease = blob.acquire_lease() # Act create_resp = blob.upload_blob(data, lease=lease) # Assert output = blob.download_blob(lease=lease) self.assertEqual(b"".join(list(output)), data) self.assertEqual(output.properties.etag, create_resp.get('etag')) self.assertEqual(output.properties.last_modified, create_resp.get('last_modified')) def test_create_blob_from_bytes_with_metadata(self): # parallel tests introduce random order of requests, can only run live if TestMode.need_recording_file(self.test_mode): return # Arrange blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = self.get_random_bytes(LARGE_BLOB_SIZE) metadata = {'hello': 'world', 'number': '42'} # Act blob.upload_blob(data, metadata=metadata) # Assert md = blob.get_blob_properties().metadata self.assertDictEqual(md, metadata) def test_create_blob_from_bytes_with_properties(self): # parallel tests introduce random order of requests, can only run live if TestMode.need_recording_file(self.test_mode): return # Arrange blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = self.get_random_bytes(LARGE_BLOB_SIZE) # Act content_settings = ContentSettings(content_type='image/png', content_language='spanish') blob.upload_blob(data, content_settings=content_settings) # Assert self.assertBlobEqual(self.container_name, blob_name, data) properties = blob.get_blob_properties() self.assertEqual(properties.content_settings.content_type, content_settings.content_type) self.assertEqual(properties.content_settings.content_language, content_settings.content_language) def test_create_blob_from_bytes_with_progress(self): # parallel tests introduce random order of requests, can only run live if TestMode.need_recording_file(self.test_mode): return # Arrange blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = self.get_random_bytes(LARGE_BLOB_SIZE) # Act progress = [] def callback(response): current = response.context['upload_stream_current'] total = response.context['data_stream_total'] if current is not None: progress.append((current, total)) create_resp = blob.upload_blob(data, raw_response_hook=callback) props = blob.get_blob_properties() # Assert self.assertBlobEqual(self.container_name, blob_name, data) self.assert_upload_progress(len(data), self.config.max_block_size, progress) self.assertEqual(props.etag, create_resp.get('etag')) self.assertEqual(props.last_modified, create_resp.get('last_modified')) def test_create_blob_from_bytes_with_index(self): # parallel tests introduce random order of requests, can only run live if TestMode.need_recording_file(self.test_mode): return # Arrange blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = self.get_random_bytes(LARGE_BLOB_SIZE) # Act blob.upload_blob(data[3:]) # Assert self.assertEqual(data[3:], b"".join(list(blob.download_blob()))) @record def test_create_blob_from_bytes_with_index_and_count(self): # Arrange blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = self.get_random_bytes(LARGE_BLOB_SIZE) # Act blob.upload_blob(data[3:], length=5) # Assert self.assertEqual(data[3:8], b"".join(list(blob.download_blob()))) @record def test_create_blob_from_bytes_with_index_and_count_and_properties(self): # Arrange blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = self.get_random_bytes(LARGE_BLOB_SIZE) # Act content_settings = ContentSettings(content_type='image/png', content_language='spanish') blob.upload_blob(data[3:], length=5, content_settings=content_settings) # Assert self.assertEqual(data[3:8], b"".join(list(blob.download_blob()))) properties = blob.get_blob_properties() self.assertEqual(properties.content_settings.content_type, content_settings.content_type) self.assertEqual(properties.content_settings.content_language, content_settings.content_language) @record def test_create_blob_from_bytes_non_parallel(self): # Arrange blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = self.get_random_bytes(LARGE_BLOB_SIZE) # Act blob.upload_blob(data, length=LARGE_BLOB_SIZE, max_connections=1) # Assert self.assertBlobEqual(self.container_name, blob.blob_name, data) def test_create_blob_from_path(self): # parallel tests introduce random order of requests, can only run live if TestMode.need_recording_file(self.test_mode): return # Arrange blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = self.get_random_bytes(LARGE_BLOB_SIZE) with open(FILE_PATH, 'wb') as stream: stream.write(data) # Act with open(FILE_PATH, 'rb') as stream: create_resp = blob.upload_blob(stream) props = blob.get_blob_properties() # Assert self.assertBlobEqual(self.container_name, blob_name, data) self.assertEqual(props.etag, create_resp.get('etag')) self.assertEqual(props.last_modified, create_resp.get('last_modified')) @record def test_create_blob_from_path_non_parallel(self): # Arrange blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = self.get_random_bytes(100) with open(FILE_PATH, 'wb') as stream: stream.write(data) # Act with open(FILE_PATH, 'rb') as stream: create_resp = blob.upload_blob(stream, length=100, max_connections=1) props = blob.get_blob_properties() # Assert self.assertBlobEqual(self.container_name, blob_name, data) self.assertEqual(props.etag, create_resp.get('etag')) self.assertEqual(props.last_modified, create_resp.get('last_modified')) def test_create_blob_from_path_with_progress(self): # parallel tests introduce random order of requests, can only run live if TestMode.need_recording_file(self.test_mode): return # Arrange blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = self.get_random_bytes(LARGE_BLOB_SIZE) with open(FILE_PATH, 'wb') as stream: stream.write(data) # Act progress = [] def callback(response): current = response.context['upload_stream_current'] total = response.context['data_stream_total'] if current is not None: progress.append((current, total)) with open(FILE_PATH, 'rb') as stream: blob.upload_blob(stream, raw_response_hook=callback) # Assert self.assertBlobEqual(self.container_name, blob_name, data) self.assert_upload_progress(len(data), self.config.max_block_size, progress) def test_create_blob_from_path_with_properties(self): # parallel tests introduce random order of requests, can only run live if TestMode.need_recording_file(self.test_mode): return # Arrange blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = self.get_random_bytes(LARGE_BLOB_SIZE) with open(FILE_PATH, 'wb') as stream: stream.write(data) # Act content_settings = ContentSettings(content_type='image/png', content_language='spanish') with open(FILE_PATH, 'rb') as stream: blob.upload_blob(stream, content_settings=content_settings) # Assert self.assertBlobEqual(self.container_name, blob_name, data) properties = blob.get_blob_properties() self.assertEqual(properties.content_settings.content_type, content_settings.content_type) self.assertEqual(properties.content_settings.content_language, content_settings.content_language) def test_create_blob_from_stream_chunked_upload(self): # parallel tests introduce random order of requests, can only run live if TestMode.need_recording_file(self.test_mode): return # Arrange blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = self.get_random_bytes(LARGE_BLOB_SIZE) with open(FILE_PATH, 'wb') as stream: stream.write(data) # Act with open(FILE_PATH, 'rb') as stream: create_resp = blob.upload_blob(stream) props = blob.get_blob_properties() # Assert self.assertBlobEqual(self.container_name, blob_name, data) self.assertEqual(props.etag, create_resp.get('etag')) self.assertEqual(props.last_modified, create_resp.get('last_modified')) def test_create_blob_from_stream_non_seekable_chunked_upload_known_size( self): # parallel tests introduce random order of requests, can only run live if TestMode.need_recording_file(self.test_mode): return # Arrange blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = self.get_random_bytes(LARGE_BLOB_SIZE) blob_size = len(data) - 66 with open(FILE_PATH, 'wb') as stream: stream.write(data) # Act with open(FILE_PATH, 'rb') as stream: non_seekable_file = StorageBlockBlobTest.NonSeekableFile(stream) blob.upload_blob(non_seekable_file, length=blob_size, max_connections=1) # Assert self.assertBlobEqual(self.container_name, blob_name, data[:blob_size]) def test_create_blob_from_stream_non_seekable_chunked_upload_unknown_size( self): # parallel tests introduce random order of requests, can only run live if TestMode.need_recording_file(self.test_mode): return # Arrange blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = self.get_random_bytes(LARGE_BLOB_SIZE) with open(FILE_PATH, 'wb') as stream: stream.write(data) # Act with open(FILE_PATH, 'rb') as stream: non_seekable_file = StorageBlockBlobTest.NonSeekableFile(stream) blob.upload_blob(non_seekable_file, max_connections=1) # Assert self.assertBlobEqual(self.container_name, blob_name, data) def test_create_blob_from_stream_with_progress_chunked_upload(self): # parallel tests introduce random order of requests, can only run live if TestMode.need_recording_file(self.test_mode): return # Arrange blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = self.get_random_bytes(LARGE_BLOB_SIZE) with open(FILE_PATH, 'wb') as stream: stream.write(data) # Act progress = [] def callback(response): current = response.context['upload_stream_current'] total = response.context['data_stream_total'] if current is not None: progress.append((current, total)) with open(FILE_PATH, 'rb') as stream: blob.upload_blob(stream, raw_response_hook=callback) # Assert self.assertBlobEqual(self.container_name, blob_name, data) self.assert_upload_progress(len(data), self.config.max_block_size, progress) def test_create_blob_from_stream_chunked_upload_with_count(self): # parallel tests introduce random order of requests, can only run live if TestMode.need_recording_file(self.test_mode): return # Arrange blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = self.get_random_bytes(LARGE_BLOB_SIZE) with open(FILE_PATH, 'wb') as stream: stream.write(data) # Act blob_size = len(data) - 301 with open(FILE_PATH, 'rb') as stream: resp = blob.upload_blob(stream, length=blob_size) # Assert self.assertBlobEqual(self.container_name, blob_name, data[:blob_size]) def test_create_blob_from_stream_chunked_upload_with_count_and_properties( self): # parallel tests introduce random order of requests, can only run live if TestMode.need_recording_file(self.test_mode): return # Arrange blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = self.get_random_bytes(LARGE_BLOB_SIZE) with open(FILE_PATH, 'wb') as stream: stream.write(data) # Act content_settings = ContentSettings(content_type='image/png', content_language='spanish') blob_size = len(data) - 301 with open(FILE_PATH, 'rb') as stream: blob.upload_blob(stream, length=blob_size, content_settings=content_settings) # Assert self.assertBlobEqual(self.container_name, blob_name, data[:blob_size]) properties = blob.get_blob_properties() self.assertEqual(properties.content_settings.content_type, content_settings.content_type) self.assertEqual(properties.content_settings.content_language, content_settings.content_language) def test_create_blob_from_stream_chunked_upload_with_properties(self): # parallel tests introduce random order of requests, can only run live if TestMode.need_recording_file(self.test_mode): return # Arrange blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = self.get_random_bytes(LARGE_BLOB_SIZE) with open(FILE_PATH, 'wb') as stream: stream.write(data) # Act content_settings = ContentSettings(content_type='image/png', content_language='spanish') with open(FILE_PATH, 'rb') as stream: blob.upload_blob(stream, content_settings=content_settings) # Assert self.assertBlobEqual(self.container_name, blob_name, data) properties = blob.get_blob_properties() self.assertEqual(properties.content_settings.content_type, content_settings.content_type) self.assertEqual(properties.content_settings.content_language, content_settings.content_language) @record def test_create_blob_from_text(self): # Arrange blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) text = u'hello 啊齄丂狛狜 world' data = text.encode('utf-8') # Act create_resp = blob.upload_blob(text) props = blob.get_blob_properties() # Assert self.assertBlobEqual(self.container_name, blob_name, data) self.assertEqual(props.etag, create_resp.get('etag')) self.assertEqual(props.last_modified, create_resp.get('last_modified')) @record def test_create_blob_from_text_with_encoding(self): # Arrange blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) text = u'hello 啊齄丂狛狜 world' data = text.encode('utf-16') # Act blob.upload_blob(text, encoding='utf-16') # Assert self.assertBlobEqual(self.container_name, blob_name, data) @record def test_create_blob_from_text_with_encoding_and_progress(self): # Arrange blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) text = u'hello 啊齄丂狛狜 world' data = text.encode('utf-16') # Act progress = [] def callback(response): current = response.context['upload_stream_current'] total = response.context['data_stream_total'] if current is not None: progress.append((current, total)) blob.upload_blob(text, encoding='utf-16', raw_response_hook=callback) # Assert self.assertBlobEqual(self.container_name, blob_name, data) self.assert_upload_progress(len(data), self.config.max_block_size, progress) def test_create_blob_from_text_chunked_upload(self): # parallel tests introduce random order of requests, can only run live if TestMode.need_recording_file(self.test_mode): return # Arrange blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = self.get_random_text_data(LARGE_BLOB_SIZE) encoded_data = data.encode('utf-8') # Act blob.upload_blob(data) # Assert self.assertBlobEqual(self.container_name, blob_name, encoded_data) # Assert self.assertBlobEqual(self.container_name, blob_name, encoded_data) @record def test_create_blob_with_md5(self): # Arrange blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = b'hello world' # Act blob.upload_blob(data, validate_content=True) # Assert def test_create_blob_with_md5_chunked(self): # parallel tests introduce random order of requests, can only run live if TestMode.need_recording_file(self.test_mode): return # Arrange blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = self.get_random_bytes(LARGE_BLOB_SIZE) # Act blob.upload_blob(data, validate_content=True)
class StorageBlockBlobTest(StorageTestCase): def _setup(self, storage_account, key): # test chunking functionality by reducing the size of each chunk, # otherwise the tests would take too long to execute self.bsc = BlobServiceClient( self.account_url(storage_account, "blob"), credential=key, connection_data_block_size=4 * 1024, max_single_put_size=32 * 1024, max_block_size=4 * 1024) self.config = self.bsc._config self.container_name = self.get_resource_name('utcontainer') if self.is_live: self.bsc.create_container(self.container_name) def _teardown(self, FILE_PATH): if os.path.isfile(FILE_PATH): try: os.remove(FILE_PATH) except: pass #--Helpers----------------------------------------------------------------- def _get_blob_reference(self): return self.get_resource_name(TEST_BLOB_PREFIX) def _create_blob(self): blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) blob.upload_blob(b'') return blob def assertBlobEqual(self, container_name, blob_name, expected_data): blob = self.bsc.get_blob_client(container_name, blob_name) actual_data = blob.download_blob() self.assertEqual(actual_data.readall(), expected_data) class NonSeekableFile(object): def __init__(self, wrapped_file): self.wrapped_file = wrapped_file def write(self, data): self.wrapped_file.write(data) def read(self, count): return self.wrapped_file.read(count) #--Test cases for block blobs -------------------------------------------- @GlobalStorageAccountPreparer() def test_put_block(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) blob = self._create_blob() # Act for i in range(5): headers = blob.stage_block(i, 'block {0}'.format(i).encode('utf-8')) self.assertIn('content_crc64', headers) # Assert @GlobalStorageAccountPreparer() def test_put_block_with_response(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account.name, storage_account_key) blob = self._create_blob() def return_response(resp, _, headers): return (resp, headers) # Act resp, headers = blob.stage_block(0, 'block 0', cls=return_response) # Assert self.assertEqual(201, resp.status_code) self.assertIn('x-ms-content-crc64', headers) @GlobalStorageAccountPreparer() def test_put_block_unicode(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) blob = self._create_blob() # Act headers = blob.stage_block('1', u'啊齄丂狛狜') self.assertIn('content_crc64', headers) # Assert @GlobalStorageAccountPreparer() def test_put_block_with_md5(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) blob = self._create_blob() # Act blob.stage_block(1, b'block', validate_content=True) # Assert @GlobalStorageAccountPreparer() def test_put_block_list(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) blob.stage_block('1', b'AAA') blob.stage_block('2', b'BBB') blob.stage_block('3', b'CCC') # Act block_list = [BlobBlock(block_id='1'), BlobBlock(block_id='2'), BlobBlock(block_id='3')] put_block_list_resp = blob.commit_block_list(block_list) # Assert content = blob.download_blob() self.assertEqual(content.readall(), b'AAABBBCCC') self.assertEqual(content.properties.etag, put_block_list_resp.get('etag')) self.assertEqual(content.properties.last_modified, put_block_list_resp.get('last_modified')) @GlobalStorageAccountPreparer() def test_put_block_list_invalid_block_id(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) blob.stage_block('1', b'AAA') blob.stage_block('2', b'BBB') blob.stage_block('3', b'CCC') # Act try: block_list = [BlobBlock(block_id='1'), BlobBlock(block_id='2'), BlobBlock(block_id='4')] blob.commit_block_list(block_list) self.fail() except HttpResponseError as e: self.assertGreaterEqual(str(e).find('specified block list is invalid'), 0) # Assert @GlobalStorageAccountPreparer() def test_put_block_list_with_md5(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) blob.stage_block('1', b'AAA') blob.stage_block('2', b'BBB') blob.stage_block('3', b'CCC') # Act block_list = [BlobBlock(block_id='1'), BlobBlock(block_id='2'), BlobBlock(block_id='3')] blob.commit_block_list(block_list, validate_content=True) # Assert @GlobalStorageAccountPreparer() def test_put_block_list_with_blob_tier_specified(self, resource_group, location, storage_account, storage_account_key): # Arrange self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob_client = self.bsc.get_blob_client(self.container_name, blob_name) blob_client.stage_block('1', b'AAA') blob_client.stage_block('2', b'BBB') blob_client.stage_block('3', b'CCC') blob_tier = StandardBlobTier.Cool # Act block_list = [BlobBlock(block_id='1'), BlobBlock(block_id='2'), BlobBlock(block_id='3')] blob_client.commit_block_list(block_list, standard_blob_tier=blob_tier) # Assert blob_properties = blob_client.get_blob_properties() self.assertEqual(blob_properties.blob_tier, blob_tier) @GlobalStorageAccountPreparer() def test_get_block_list_no_blocks(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) blob = self._create_blob() # Act block_list = blob.get_block_list('all') # Assert self.assertIsNotNone(block_list) self.assertEqual(len(block_list[1]), 0) self.assertEqual(len(block_list[0]), 0) @GlobalStorageAccountPreparer() def test_get_block_list_uncommitted_blocks(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) blob.stage_block('1', b'AAA') blob.stage_block('2', b'BBB') blob.stage_block('3', b'CCC') # Act block_list = blob.get_block_list('uncommitted') # Assert self.assertIsNotNone(block_list) self.assertEqual(len(block_list), 2) self.assertEqual(len(block_list[1]), 3) self.assertEqual(len(block_list[0]), 0) self.assertEqual(block_list[1][0].id, '1') self.assertEqual(block_list[1][0].size, 3) self.assertEqual(block_list[1][1].id, '2') self.assertEqual(block_list[1][1].size, 3) self.assertEqual(block_list[1][2].id, '3') self.assertEqual(block_list[1][2].size, 3) @GlobalStorageAccountPreparer() def test_get_block_list_committed_blocks(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) blob.stage_block('1', b'AAA') blob.stage_block('2', b'BBB') blob.stage_block('3', b'CCC') block_list = [BlobBlock(block_id='1'), BlobBlock(block_id='2'), BlobBlock(block_id='3')] blob.commit_block_list(block_list) # Act block_list = blob.get_block_list('committed') # Assert self.assertIsNotNone(block_list) self.assertEqual(len(block_list), 2) self.assertEqual(len(block_list[1]), 0) self.assertEqual(len(block_list[0]), 3) self.assertEqual(block_list[0][0].id, '1') self.assertEqual(block_list[0][0].size, 3) self.assertEqual(block_list[0][1].id, '2') self.assertEqual(block_list[0][1].size, 3) self.assertEqual(block_list[0][2].id, '3') self.assertEqual(block_list[0][2].size, 3) @GlobalStorageAccountPreparer() def test_create_small_block_blob_with_no_overwrite(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data1 = b'hello world' data2 = b'hello second world' # Act create_resp = blob.upload_blob(data1, overwrite=True) with self.assertRaises(ResourceExistsError): blob.upload_blob(data2, overwrite=False) props = blob.get_blob_properties() # Assert self.assertBlobEqual(self.container_name, blob_name, data1) self.assertEqual(props.etag, create_resp.get('etag')) self.assertEqual(props.last_modified, create_resp.get('last_modified')) self.assertEqual(props.blob_type, BlobType.BlockBlob) @GlobalStorageAccountPreparer() def test_create_small_block_blob_with_overwrite(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data1 = b'hello world' data2 = b'hello second world' # Act create_resp = blob.upload_blob(data1, overwrite=True) update_resp = blob.upload_blob(data2, overwrite=True) props = blob.get_blob_properties() # Assert self.assertBlobEqual(self.container_name, blob_name, data2) self.assertEqual(props.etag, update_resp.get('etag')) self.assertEqual(props.last_modified, update_resp.get('last_modified')) self.assertEqual(props.blob_type, BlobType.BlockBlob) @GlobalStorageAccountPreparer() def test_create_large_block_blob_with_no_overwrite(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data1 = self.get_random_bytes(LARGE_BLOB_SIZE) data2 = self.get_random_bytes(LARGE_BLOB_SIZE) # Act create_resp = blob.upload_blob(data1, overwrite=True, metadata={'blobdata': 'data1'}) with self.assertRaises(ResourceExistsError): blob.upload_blob(data2, overwrite=False, metadata={'blobdata': 'data2'}) props = blob.get_blob_properties() # Assert self.assertBlobEqual(self.container_name, blob_name, data1) self.assertEqual(props.etag, create_resp.get('etag')) self.assertEqual(props.last_modified, create_resp.get('last_modified')) self.assertEqual(props.blob_type, BlobType.BlockBlob) self.assertEqual(props.metadata, {'blobdata': 'data1'}) self.assertEqual(props.size, LARGE_BLOB_SIZE) @GlobalStorageAccountPreparer() def test_create_large_block_blob_with_overwrite(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data1 = self.get_random_bytes(LARGE_BLOB_SIZE) data2 = self.get_random_bytes(LARGE_BLOB_SIZE + 512) # Act create_resp = blob.upload_blob(data1, overwrite=True, metadata={'blobdata': 'data1'}) update_resp = blob.upload_blob(data2, overwrite=True, metadata={'blobdata': 'data2'}) props = blob.get_blob_properties() # Assert self.assertBlobEqual(self.container_name, blob_name, data2) self.assertEqual(props.etag, update_resp.get('etag')) self.assertEqual(props.last_modified, update_resp.get('last_modified')) self.assertEqual(props.blob_type, BlobType.BlockBlob) self.assertEqual(props.metadata, {'blobdata': 'data2'}) self.assertEqual(props.size, LARGE_BLOB_SIZE + 512) @GlobalStorageAccountPreparer() def test_create_blob_from_bytes_single_put(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = b'hello world' # Act create_resp = blob.upload_blob(data) props = blob.get_blob_properties() # Assert self.assertBlobEqual(self.container_name, blob_name, data) self.assertEqual(props.etag, create_resp.get('etag')) self.assertEqual(props.last_modified, create_resp.get('last_modified')) @GlobalStorageAccountPreparer() def test_create_blob_from_0_bytes(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = b'' # Act create_resp = blob.upload_blob(data) props = blob.get_blob_properties() # Assert self.assertBlobEqual(self.container_name, blob_name, data) self.assertEqual(props.etag, create_resp.get('etag')) self.assertEqual(props.last_modified, create_resp.get('last_modified')) @GlobalStorageAccountPreparer() def test_create_from_bytes_blob_unicode(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = u'hello world' # Act create_resp = blob.upload_blob(data) props = blob.get_blob_properties() # Assert self.assertBlobEqual(self.container_name, blob_name, data) self.assertEqual(props.etag, create_resp.get('etag')) self.assertEqual(props.last_modified, create_resp.get('last_modified')) @GlobalStorageAccountPreparer() def test_create_from_bytes_blob_unicode(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) # Act data = u'hello world' create_resp = blob.upload_blob(data) props = blob.get_blob_properties() # Assert self.assertBlobEqual(self.container_name, blob_name, data.encode('utf-8')) self.assertEqual(props.etag, create_resp.get('etag')) self.assertEqual(props.last_modified, create_resp.get('last_modified')) @pytest.mark.live_test_only @GlobalStorageAccountPreparer() def test_create_from_bytes_blob_with_lease_id(self, resource_group, location, storage_account, storage_account_key): # parallel tests introduce random order of requests, can only run live self._setup(storage_account, storage_account_key) blob = self._create_blob() data = self.get_random_bytes(LARGE_BLOB_SIZE) lease = blob.acquire_lease() # Act create_resp = blob.upload_blob(data, lease=lease) # Assert output = blob.download_blob(lease=lease) self.assertEqual(output.readall(), data) self.assertEqual(output.properties.etag, create_resp.get('etag')) self.assertEqual(output.properties.last_modified, create_resp.get('last_modified')) @pytest.mark.live_test_only @GlobalStorageAccountPreparer() def test_create_blob_from_bytes_with_metadata(self, resource_group, location, storage_account, storage_account_key): # parallel tests introduce random order of requests, can only run live self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = self.get_random_bytes(LARGE_BLOB_SIZE) metadata = {'hello': 'world', 'number': '42'} # Act blob.upload_blob(data, metadata=metadata) # Assert md = blob.get_blob_properties().metadata self.assertDictEqual(md, metadata) @pytest.mark.live_test_only @GlobalStorageAccountPreparer() def test_create_blob_from_bytes_with_properties(self, resource_group, location, storage_account, storage_account_key): # parallel tests introduce random order of requests, can only run live self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = self.get_random_bytes(LARGE_BLOB_SIZE) # Act content_settings=ContentSettings( content_type='image/png', content_language='spanish') blob.upload_blob(data, content_settings=content_settings) # Assert self.assertBlobEqual(self.container_name, blob_name, data) properties = blob.get_blob_properties() self.assertEqual(properties.content_settings.content_type, content_settings.content_type) self.assertEqual(properties.content_settings.content_language, content_settings.content_language) @pytest.mark.live_test_only @GlobalStorageAccountPreparer() def test_create_blob_from_bytes_with_progress(self, resource_group, location, storage_account, storage_account_key): # parallel tests introduce random order of requests, can only run live self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = self.get_random_bytes(LARGE_BLOB_SIZE) # Act progress = [] def callback(response): current = response.context['upload_stream_current'] total = response.context['data_stream_total'] if current is not None: progress.append((current, total)) create_resp = blob.upload_blob(data, raw_response_hook=callback) props = blob.get_blob_properties() # Assert self.assertBlobEqual(self.container_name, blob_name, data) self.assert_upload_progress(len(data), self.config.max_block_size, progress) self.assertEqual(props.etag, create_resp.get('etag')) self.assertEqual(props.last_modified, create_resp.get('last_modified')) @pytest.mark.live_test_only @GlobalStorageAccountPreparer() def test_create_blob_from_bytes_with_index(self, resource_group, location, storage_account, storage_account_key): # parallel tests introduce random order of requests, can only run live self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = self.get_random_bytes(LARGE_BLOB_SIZE) # Act blob.upload_blob(data[3:]) # Assert self.assertEqual(data[3:], blob.download_blob().readall()) @GlobalStorageAccountPreparer() def test_create_blob_from_bytes_with_index_and_count(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = self.get_random_bytes(LARGE_BLOB_SIZE) # Act blob.upload_blob(data[3:], length=5) # Assert self.assertEqual(data[3:8], blob.download_blob().readall()) @GlobalStorageAccountPreparer() def test_create_blob_from_bytes_with_index_and_count_and_properties(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = self.get_random_bytes(LARGE_BLOB_SIZE) # Act content_settings=ContentSettings( content_type='image/png', content_language='spanish') blob.upload_blob(data[3:], length=5, content_settings=content_settings) # Assert self.assertEqual(data[3:8], blob.download_blob().readall()) properties = blob.get_blob_properties() self.assertEqual(properties.content_settings.content_type, content_settings.content_type) self.assertEqual(properties.content_settings.content_language, content_settings.content_language) @GlobalStorageAccountPreparer() def test_create_blob_from_bytes_non_parallel(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = self.get_random_bytes(LARGE_BLOB_SIZE) # Act blob.upload_blob(data, length=LARGE_BLOB_SIZE, max_concurrency=1) # Assert self.assertBlobEqual(self.container_name, blob.blob_name, data) @GlobalStorageAccountPreparer() def test_create_blob_from_bytes_with_blob_tier_specified(self, resource_group, location, storage_account, storage_account_key): # Arrange self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob_client = self.bsc.get_blob_client(self.container_name, blob_name) data = b'hello world' blob_tier = StandardBlobTier.Cool # Act blob_client.upload_blob(data, standard_blob_tier=blob_tier) blob_properties = blob_client.get_blob_properties() # Assert self.assertEqual(blob_properties.blob_tier, blob_tier) @pytest.mark.live_test_only @GlobalStorageAccountPreparer() def test_create_blob_from_path(self, resource_group, location, storage_account, storage_account_key): # parallel tests introduce random order of requests, can only run live self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = self.get_random_bytes(LARGE_BLOB_SIZE) FILE_PATH = 'create_blob_from_input.temp.{}.dat'.format(str(uuid.uuid4())) with open(FILE_PATH, 'wb') as stream: stream.write(data) # Act with open(FILE_PATH, 'rb') as stream: create_resp = blob.upload_blob(stream) props = blob.get_blob_properties() # Assert self.assertBlobEqual(self.container_name, blob_name, data) self.assertEqual(props.etag, create_resp.get('etag')) self.assertEqual(props.last_modified, create_resp.get('last_modified')) self._teardown(FILE_PATH) @GlobalStorageAccountPreparer() def test_create_blob_from_path_non_parallel(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = self.get_random_bytes(100) FILE_PATH = 'create_blob_from_path_non_par.temp.{}.dat'.format(str(uuid.uuid4())) with open(FILE_PATH, 'wb') as stream: stream.write(data) # Act with open(FILE_PATH, 'rb') as stream: create_resp = blob.upload_blob(stream, length=100, max_concurrency=1) props = blob.get_blob_properties() # Assert self.assertBlobEqual(self.container_name, blob_name, data) self.assertEqual(props.etag, create_resp.get('etag')) self.assertEqual(props.last_modified, create_resp.get('last_modified')) self._teardown(FILE_PATH) @GlobalStorageAccountPreparer() def test_upload_blob_from_path_non_parallel_with_standard_blob_tier(self, resource_group, location, storage_account, storage_account_key): # Arrange self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = self.get_random_bytes(100) FILE_PATH = '_path_non_parallel_with_standard_blob.temp.{}.dat'.format(str(uuid.uuid4())) with open(FILE_PATH, 'wb') as stream: stream.write(data) blob_tier = StandardBlobTier.Cool # Act with open(FILE_PATH, 'rb') as stream: blob.upload_blob(stream, length=100, max_concurrency=1, standard_blob_tier=blob_tier) props = blob.get_blob_properties() # Assert self.assertEqual(props.blob_tier, blob_tier) self._teardown(FILE_PATH) @pytest.mark.live_test_only @GlobalStorageAccountPreparer() def test_create_blob_from_path_with_progress(self, resource_group, location, storage_account, storage_account_key): # parallel tests introduce random order of requests, can only run live self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = self.get_random_bytes(LARGE_BLOB_SIZE) FILE_PATH = 'create_blob_from_path_with_progr.temp.{}.dat'.format(str(uuid.uuid4())) with open(FILE_PATH, 'wb') as stream: stream.write(data) # Act progress = [] def callback(response): current = response.context['upload_stream_current'] total = response.context['data_stream_total'] if current is not None: progress.append((current, total)) with open(FILE_PATH, 'rb') as stream: blob.upload_blob(stream, raw_response_hook=callback) # Assert self.assertBlobEqual(self.container_name, blob_name, data) self.assert_upload_progress(len(data), self.config.max_block_size, progress) self._teardown(FILE_PATH) @pytest.mark.live_test_only @GlobalStorageAccountPreparer() def test_create_blob_from_path_with_properties(self, resource_group, location, storage_account, storage_account_key): # parallel tests introduce random order of requests, can only run live self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = self.get_random_bytes(LARGE_BLOB_SIZE) FILE_PATH = 'blob_from_path_with_properties.temp.{}.dat'.format(str(uuid.uuid4())) with open(FILE_PATH, 'wb') as stream: stream.write(data) # Act content_settings=ContentSettings( content_type='image/png', content_language='spanish') with open(FILE_PATH, 'rb') as stream: blob.upload_blob(stream, content_settings=content_settings) # Assert self.assertBlobEqual(self.container_name, blob_name, data) properties = blob.get_blob_properties() self.assertEqual(properties.content_settings.content_type, content_settings.content_type) self.assertEqual(properties.content_settings.content_language, content_settings.content_language) self._teardown(FILE_PATH) @pytest.mark.live_test_only @GlobalStorageAccountPreparer() def test_create_blob_from_stream_chunked_upload(self, resource_group, location, storage_account, storage_account_key): # parallel tests introduce random order of requests, can only run live self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = self.get_random_bytes(LARGE_BLOB_SIZE) FILE_PATH = 'blob_from_stream_chunked_up.temp.{}.dat'.format(str(uuid.uuid4())) with open(FILE_PATH, 'wb') as stream: stream.write(data) # Act with open(FILE_PATH, 'rb') as stream: create_resp = blob.upload_blob(stream) props = blob.get_blob_properties() # Assert self.assertBlobEqual(self.container_name, blob_name, data) self.assertEqual(props.etag, create_resp.get('etag')) self.assertEqual(props.last_modified, create_resp.get('last_modified')) self._teardown(FILE_PATH) @pytest.mark.live_test_only @GlobalStorageAccountPreparer() def test_create_frm_stream_nonseek_chunk_upld_knwn_size(self, resource_group, location, storage_account, storage_account_key): # parallel tests introduce random order of requests, can only run live self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = self.get_random_bytes(LARGE_BLOB_SIZE) blob_size = len(data) - 66 FILE_PATH = 'stream_nonseek_chunk_upld_knwn_size.temp.{}.dat'.format(str(uuid.uuid4())) with open(FILE_PATH, 'wb') as stream: stream.write(data) # Act with open(FILE_PATH, 'rb') as stream: non_seekable_file = StorageBlockBlobTest.NonSeekableFile(stream) blob.upload_blob(non_seekable_file, length=blob_size, max_concurrency=1) # Assert self.assertBlobEqual(self.container_name, blob_name, data[:blob_size]) self._teardown(FILE_PATH) @pytest.mark.live_test_only @GlobalStorageAccountPreparer() def test_create_from_stream_nonseek_chunk_upld_unkwn_size(self, resource_group, location, storage_account, storage_account_key): # parallel tests introduce random order of requests, can only run live self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = self.get_random_bytes(LARGE_BLOB_SIZE) FILE_PATH = 'stream_nonseek_chunk_upld.temp.{}.dat'.format(str(uuid.uuid4())) with open(FILE_PATH, 'wb') as stream: stream.write(data) # Act with open(FILE_PATH, 'rb') as stream: non_seekable_file = StorageBlockBlobTest.NonSeekableFile(stream) blob.upload_blob(non_seekable_file, max_concurrency=1) # Assert self.assertBlobEqual(self.container_name, blob_name, data) self._teardown(FILE_PATH) @pytest.mark.live_test_only @GlobalStorageAccountPreparer() def test_create_blob_from_stream_with_progress_chunked_upload(self, resource_group, location, storage_account, storage_account_key): # parallel tests introduce random order of requests, can only run live self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = self.get_random_bytes(LARGE_BLOB_SIZE) FILE_PATH = 'stream_with_progress_chunked.temp.{}.dat'.format(str(uuid.uuid4())) with open(FILE_PATH, 'wb') as stream: stream.write(data) # Act progress = [] def callback(response): current = response.context['upload_stream_current'] total = response.context['data_stream_total'] if current is not None: progress.append((current, total)) with open(FILE_PATH, 'rb') as stream: blob.upload_blob(stream, raw_response_hook=callback) # Assert self.assertBlobEqual(self.container_name, blob_name, data) self.assert_upload_progress(len(data), self.config.max_block_size, progress) self._teardown(FILE_PATH) @pytest.mark.live_test_only @GlobalStorageAccountPreparer() def test_create_blob_from_stream_chunked_upload_with_count(self, resource_group, location, storage_account, storage_account_key): # parallel tests introduce random order of requests, can only run live self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = self.get_random_bytes(LARGE_BLOB_SIZE) FILE_PATH = 'chunked_upload_with_count.temp.{}.dat'.format(str(uuid.uuid4())) with open(FILE_PATH, 'wb') as stream: stream.write(data) # Act blob_size = len(data) - 301 with open(FILE_PATH, 'rb') as stream: resp = blob.upload_blob(stream, length=blob_size) # Assert self.assertBlobEqual(self.container_name, blob_name, data[:blob_size]) self._teardown(FILE_PATH) @pytest.mark.live_test_only @GlobalStorageAccountPreparer() def test_create_from_stream_chunk_upload_with_cntandrops(self, resource_group, location, storage_account, storage_account_key): # parallel tests introduce random order of requests, can only run live self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = self.get_random_bytes(LARGE_BLOB_SIZE) FILE_PATH = 'from_stream_chunk_upload_with_cntandrops.temp.{}.dat'.format(str(uuid.uuid4())) with open(FILE_PATH, 'wb') as stream: stream.write(data) # Act content_settings=ContentSettings( content_type='image/png', content_language='spanish') blob_size = len(data) - 301 with open(FILE_PATH, 'rb') as stream: blob.upload_blob(stream, length=blob_size, content_settings=content_settings) # Assert self.assertBlobEqual(self.container_name, blob_name, data[:blob_size]) properties = blob.get_blob_properties() self.assertEqual(properties.content_settings.content_type, content_settings.content_type) self.assertEqual(properties.content_settings.content_language, content_settings.content_language) self._teardown(FILE_PATH) @pytest.mark.live_test_only @GlobalStorageAccountPreparer() def test_create_blob_from_stream_chnked_upload_with_properties(self, resource_group, location, storage_account, storage_account_key): # parallel tests introduce random order of requests, can only run live self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = self.get_random_bytes(LARGE_BLOB_SIZE) FILE_PATH = 'chnked_upload_with_properti.temp.{}.dat'.format(str(uuid.uuid4())) with open(FILE_PATH, 'wb') as stream: stream.write(data) # Act content_settings=ContentSettings( content_type='image/png', content_language='spanish') with open(FILE_PATH, 'rb') as stream: blob.upload_blob(stream, content_settings=content_settings) # Assert self.assertBlobEqual(self.container_name, blob_name, data) properties = blob.get_blob_properties() self.assertEqual(properties.content_settings.content_type, content_settings.content_type) self.assertEqual(properties.content_settings.content_language, content_settings.content_language) self._teardown(FILE_PATH) @pytest.mark.live_test_only @GlobalStorageAccountPreparer() def test_create_blob_from_stream_chunked_upload_with_properties(self, resource_group, location, storage_account, storage_account_key): # parallel tests introduce random order of requests, can only run live # Arrange self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = self.get_random_bytes(LARGE_BLOB_SIZE) FILE_PATH = 'blob_from_stream_chunked_upload.temp.{}.dat'.format(str(uuid.uuid4())) with open(FILE_PATH, 'wb') as stream: stream.write(data) blob_tier = StandardBlobTier.Cool # Act content_settings = ContentSettings( content_type='image/png', content_language='spanish') with open(FILE_PATH, 'rb') as stream: blob.upload_blob(stream, content_settings=content_settings, max_concurrency=2, standard_blob_tier=blob_tier) properties = blob.get_blob_properties() # Assert self.assertEqual(properties.blob_tier, blob_tier) self._teardown(FILE_PATH) @GlobalStorageAccountPreparer() def test_create_blob_from_text(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) text = u'hello 啊齄丂狛狜 world' data = text.encode('utf-8') # Act create_resp = blob.upload_blob(text) props = blob.get_blob_properties() # Assert self.assertBlobEqual(self.container_name, blob_name, data) self.assertEqual(props.etag, create_resp.get('etag')) self.assertEqual(props.last_modified, create_resp.get('last_modified')) @GlobalStorageAccountPreparer() def test_create_blob_from_text_with_encoding(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) text = u'hello 啊齄丂狛狜 world' data = text.encode('utf-16') # Act blob.upload_blob(text, encoding='utf-16') # Assert self.assertBlobEqual(self.container_name, blob_name, data) @GlobalStorageAccountPreparer() def test_create_blob_from_text_with_encoding_and_progress(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) text = u'hello 啊齄丂狛狜 world' data = text.encode('utf-16') # Act progress = [] def callback(response): current = response.context['upload_stream_current'] total = response.context['data_stream_total'] if current is not None: progress.append((current, total)) blob.upload_blob(text, encoding='utf-16', raw_response_hook=callback) # Assert self.assertBlobEqual(self.container_name, blob_name, data) self.assert_upload_progress(len(data), self.config.max_block_size, progress) @pytest.mark.live_test_only @GlobalStorageAccountPreparer() def test_create_blob_from_text_chunked_upload(self, resource_group, location, storage_account, storage_account_key): # parallel tests introduce random order of requests, can only run live self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = self.get_random_text_data(LARGE_BLOB_SIZE) encoded_data = data.encode('utf-8') # Act blob.upload_blob(data) # Assert self.assertBlobEqual(self.container_name, blob_name, encoded_data) # Assert self.assertBlobEqual(self.container_name, blob_name, encoded_data) @GlobalStorageAccountPreparer() def test_create_blob_with_md5(self, resource_group, location, storage_account, storage_account_key): self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = b'hello world' # Act blob.upload_blob(data, validate_content=True) # Assert @pytest.mark.live_test_only @GlobalStorageAccountPreparer() def test_create_blob_with_md5_chunked(self, resource_group, location, storage_account, storage_account_key): # parallel tests introduce random order of requests, can only run live self._setup(storage_account, storage_account_key) blob_name = self._get_blob_reference() blob = self.bsc.get_blob_client(self.container_name, blob_name) data = self.get_random_bytes(LARGE_BLOB_SIZE) # Act blob.upload_blob(data, validate_content=True)