def upload_third_party(self):
logger.info("uploading third-party tools from %s", self.third_party)
account_name = self.results["deploy"]["fuzz-name"]["value"]
key = self.results["deploy"]["fuzz-key"]["value"]
account_url = "https://%s.blob.core.windows.net" % account_name
client = BlobServiceClient(account_url, credential=key)
containers = [x["name"] for x in client.list_containers()]
for name in os.listdir(self.third_party):
path = os.path.join(self.third_party, name)
if not os.path.isdir(path):
continue
if name not in containers:
client.create_container(name)
expiry = datetime.utcnow() + timedelta(minutes=30)
sas = generate_container_sas(
account_name,
name,
account_key=key,
permission=ContainerSasPermissions(
read=True, write=True, delete=True, list=True
),
expiry=expiry,
)
url = "%s/%s?%s" % (account_url, name, sas)
subprocess.check_output(
[self.azcopy, "sync", path, url, "--delete-destination", "true"]
)
def upload_instance_setup(self):
logger.info("uploading instance-specific-setup from %s", self.instance_specific)
account_name = self.results["deploy"]["func-name"]["value"]
key = self.results["deploy"]["func-key"]["value"]
account_url = "https://%s.blob.core.windows.net" % account_name
client = BlobServiceClient(account_url, credential=key)
if "instance-specific-setup" not in [
x["name"] for x in client.list_containers()
]:
client.create_container("instance-specific-setup")
expiry = datetime.utcnow() + timedelta(minutes=30)
sas = generate_container_sas(
account_name,
"instance-specific-setup",
account_key=key,
permission=ContainerSasPermissions(
read=True, write=True, delete=True, list=True
),
expiry=expiry,
)
url = "%s/%s?%s" % (account_url, "instance-specific-setup", sas)
subprocess.check_output(
[
self.azcopy,
"sync",
self.instance_specific,
url,
"--delete-destination",
"true",
]
)
def azblob_file(azblob_credentials,
cloud_bucket_name,
download_gcs_public_data,
public=False):
acc_url = f"https://{azblob_credentials['storage_account']}.blob.core.windows.net"
azblob_client = BlobServiceClient(
account_url=acc_url, credential=azblob_credentials["shared_key"])
container_name = cloud_bucket_name + random_char(3).lower()
if public:
container_name += "public"
print(f"\nUpload dataset to private azure blob container {container_name}")
if container_name not in [
cntr["name"] for cntr in azblob_client.list_containers()
]:
if public:
azblob_client.create_container(name=container_name,
metadata=None,
public_access="container")
else:
azblob_client.create_container(name=container_name,
metadata=None,
public_access=None)
blob_client = azblob_client.get_blob_client(container_name, "myfile.csv")
with open(download_gcs_public_data, "r") as f:
blob_client.upload_blob(f.read(),
blob_type="BlockBlob",
overwrite=True)
yield f"{container_name}/myfile.csv"
azblob_client.delete_container(container_name)
print(
f"\nAzure Blob Container {container_name} is now marked for deletion")
def createStorageContainer(storage_name, storage_key, container_names):
'''
Create storage containers in the provided storage account. If the containers
exist, they will not be altered.
PARAMS:
storage_name : string : Name of the Azure Storage Account
storage_key : string : Access Key to the Azure Storage Account
container_names : list[string] : List of container names to create.
RETURNS:
Nothing
'''
blob_service = BlobServiceClient(account_url="https://" + storage_name +
".blob.core.windows.net/",
credential=storage_key)
for container in container_names:
try:
blob_service.create_container(container)
print("Storage container created - ", storage_name, " : ",
container)
except ResourceExistsError:
print("Storage container already exists - ", storage_name, " : ",
container)
class AzurePersistor(Persistor):
"""Store models on Azure"""
def __init__(
self, azure_container: Text, azure_account_name: Text, azure_account_key: Text
) -> None:
from azure.storage.blob import BlobServiceClient
super().__init__()
self.blob_service = BlobServiceClient(
account_url=f"https://{azure_account_name}.blob.core.windows.net/",
credential=azure_account_key,
)
self._ensure_container_exists(azure_container)
self.container_name = azure_container
def _ensure_container_exists(self, container_name: Text) -> None:
from azure.core.exceptions import ResourceExistsError
try:
self.blob_service.create_container(container_name)
except ResourceExistsError:
# no need to create the container, it already exists
pass
def _container_client(self) -> "ContainerClient":
return self.blob_service.get_container_client(self.container_name)
def list_models(self) -> List[Text]:
"""Lists models on remote storage.
Returns:
Paths to found models.
"""
try:
blob_iterator = self._container_client().list_blobs()
return [
self._model_dir_and_model_from_filename(b.name)[1]
for b in blob_iterator
]
except Exception as e:
logger.warning(f"Failed to list models azure blob storage. {e}")
return []
def _persist_tar(self, file_key: Text, tar_path: Text) -> None:
"""Uploads a model persisted in the `target_dir` to Azure."""
with open(tar_path, "rb") as data:
self._container_client().upload_blob(name=file_key, data=data)
def _retrieve_tar(self, target_filename: Text) -> None:
"""Downloads a model that has previously been persisted to Azure."""
blob_client = self._container_client().get_blob_client(target_filename)
with open(target_filename, "wb") as blob:
download_stream = blob_client.download_blob()
blob.write(download_stream.readall())
def create_dls(wait_a_sec):
credential = CustomTokenCredential()
blob_service_client = BlobServiceClient(STORAGE_URL, credential)
try:
blob_service_client.delete_container("dls")
except Exception:
pass
blob_service_client.create_container("dls")
dummy_data = pd.DataFrame.from_dict({
"key0": ["val00", "val01"],
"key1": ["val10", "val11"]
}).to_dict(orient="split")
for (data_set,
file) in itertools.product(["deep", "shallow"],
["systems", "complexes", "elements"]):
upload(
blob_service_client,
dummy_data,
f"{FOLDER_NAME}/{data_set}/{file}.json",
)
yield dummy_data
blob_service_client.delete_container("dls")
def create_storage_account_container_if_not_exists(
storage_account_client: blob.BlobServiceClient,
container_name: str) -> None:
try:
storage_account_client.create_container(container_name)
except ResourceExistsError:
pass # Container already exists, silently ignore
def add_log_export(self) -> None:
if not self.export_appinsights:
logger.info("not exporting appinsights")
return
container_name = "app-insights"
logger.info("adding appinsight log export")
account_name = self.results["deploy"]["func_name"]["value"]
key = self.results["deploy"]["func_key"]["value"]
account_url = "https://%s.blob.core.windows.net" % account_name
client = BlobServiceClient(account_url, credential=key)
if container_name not in [x["name"] for x in client.list_containers()]:
client.create_container(container_name)
expiry = datetime.utcnow() + timedelta(days=2 * 365)
# NOTE: as this is a long-lived SAS url, it should not be logged and only
# used in the the later-on export_configurations.create() call
sas = generate_container_sas(
account_name,
container_name,
account_key=key,
permission=ContainerSasPermissions(write=True),
expiry=expiry,
)
url = "%s/%s?%s" % (account_url, container_name, sas)
record_types = (
"Requests, Event, Exceptions, Metrics, PageViews, "
"PageViewPerformance, Rdd, PerformanceCounters, Availability")
req = ApplicationInsightsComponentExportRequest(
record_types=record_types,
destination_type="Blob",
is_enabled="true",
destination_address=url,
)
credential = AzureCliCredential()
app_insight_client = ApplicationInsightsManagementClient(
credential,
subscription_id=self.get_subscription_id(),
)
to_delete = []
for entry in app_insight_client.export_configurations.list(
self.resource_group, self.application_name):
if (entry.storage_name == account_name
and entry.container_name == container_name):
to_delete.append(entry.export_id)
for export_id in to_delete:
logger.info("replacing existing export: %s", export_id)
app_insight_client.export_configurations.delete(
self.resource_group, self.application_name, export_id)
app_insight_client.export_configurations.create(
self.resource_group, self.application_name, req)
def create_resource(self, name, **kwargs):
if self.is_live:
storage_account = kwargs.pop("storage_account")
storage_account_key = kwargs.pop("storage_account_key")
sas_token = generate_account_sas(
account_name=storage_account.name,
account_key=storage_account_key,
resource_types=ResourceTypes(container=True, object=True),
permission=AccountSasPermissions(create=True,
list=True,
write=True,
read=True,
add=True,
delete=True,
delete_previous_version=True),
expiry=datetime.utcnow() + timedelta(minutes=5),
)
blob_client = BlobServiceClient(
storage_account.primary_endpoints.blob, sas_token)
container = blob_client.create_container(name)
container_uri = storage_account.primary_endpoints.blob + container.container_name
self.test_class_instance.scrubber.register_name_pair(
sas_token, "redacted")
self.test_class_instance.scrubber.register_name_pair(
container_uri, "https://storage/container")
else:
sas_token = "fake-sas"
container_uri = "https://storage/container"
return {"container_uri": container_uri, "sas_token": sas_token}
def test_user_delegation_sas_for_container(self):
# SAS URL is calculated from storage key, so this test runs live only
pytest.skip("Current Framework Cannot Support OAUTH")
if TestMode.need_recording_file(self.test_mode):
return
# Arrange
token_credential = self.generate_oauth_token()
service_client = BlobServiceClient(self._get_oauth_account_url(),
credential=token_credential)
user_delegation_key = service_client.get_user_delegation_key(
datetime.utcnow(),
datetime.utcnow() + timedelta(hours=1))
container_client = service_client.create_container(
self.get_resource_name('oauthcontainer'))
token = container_client.generate_shared_access_signature(
expiry=datetime.utcnow() + timedelta(hours=1),
permission=ContainerPermissions.READ,
user_delegation_key=user_delegation_key,
account_name='emilydevtest')
blob_client = container_client.get_blob_client(
self.get_resource_name('oauthblob'))
blob_content = self.get_random_text_data(1024)
blob_client.upload_blob(blob_content, length=len(blob_content))
# Act
new_blob_client = BlobClient(blob_client.url, credential=token)
content = new_blob_client.download_blob()
# Assert
self.assertEqual(blob_content, b"".join(list(content)).decode('utf-8'))
class BlobText():
def __init__(self, account_name, account_key, sas_token, container_name):
self.account_name = account_name
self.account_key = account_key
self.sas_token = sas_token
self.container_name = container_name
# Create the BlobServiceClient object which will be used to create a container client
self.blob_service_client = BlobServiceClient(
"https://" + account_name + ".blob.core.windows.net/", account_key)
# Create the container
self.container_client = self.blob_service_client.get_container_client(
container_name)
if self.container_client is None:
self.container_client = self.blob_service_client.create_container(
container_name)
def write_text(self, blob_name, text):
data = text.encode('utf-16')
blobClient = self.container_client.get_blob_client(blob_name)
if blobClient.exists:
blobClient.delete_blob
blobClient.upload_blob(data, overwrite=True)
def read_text(self, blob_name):
blobClient = self.container_client.get_blob_client(blob_name)
if blobClient.exists:
data = blobClient.download_blob().readall()
return data.decode('utf-16')
else:
return None
def __init__(self, blob_service: BlobServiceClient, db: DatabaseProxy):
try:
self.blob_container = blob_service.create_container("images")
except ResourceExistsError as exc:
self.blob_container = blob_service.get_container_client(
container="images")
self.cosmos_container = db.create_container_if_not_exists(
id="images", partition_key=PartitionKey(path="/image/path"))
class AzurePersistor(Persistor):
"""Store models on Azure"""
def __init__(
self, azure_container: Text, azure_account_name: Text, azure_account_key: Text
) -> None:
from azure.storage.blob import BlobServiceClient
super().__init__()
self.blob_service = BlobServiceClient(
account_url=f"https://{azure_account_name}.blob.core.windows.net/",
credential=azure_account_key,
)
self._ensure_container_exists(azure_container)
self.container_name = azure_container
def _ensure_container_exists(self, container_name: Text) -> None:
from azure.core.exceptions import ResourceExistsError
try:
self.blob_service.create_container(container_name)
except ResourceExistsError:
# no need to create the container, it already exists
pass
def _container_client(self) -> "ContainerClient":
return self.blob_service.get_container_client(self.container_name)
def _persist_tar(self, file_key: Text, tar_path: Text) -> None:
"""Uploads a model persisted in the `target_dir` to Azure."""
with open(tar_path, "rb") as data:
self._container_client().upload_blob(name=file_key, data=data)
def _retrieve_tar(self, target_filename: Text) -> None:
"""Downloads a model that has previously been persisted to Azure."""
blob_client = self._container_client().get_blob_client(target_filename)
with open(target_filename, "wb") as blob:
download_stream = blob_client.download_blob()
blob.write(download_stream.readall())
def save_to_blob(self,
connection_string: str,
container: str,
path: str = None) -> str:
""" Saves the model to an azure storage blob.
The file will be located at the container and path parameter if provided, otherwise, in the root of the container."""
path = path or 'azureSDKTrackClassifier_{}_{}.model'.format(
self._language, self._service)
if 'sig=' in connection_string and 'AccountKey=' not in connection_string: # SAS signature.
service_client = BlobServiceClient(connection_string)
else:
service_client = BlobServiceClient.from_connection_string(
conn_str=connection_string)
try:
service_client.create_container(container)
except ResourceExistsError:
pass
blob_client = service_client.get_blob_client(container, path)
blob_client.upload_blob(pickle.dumps(self), overwrite=True)
return path
def add_instance_id(self) -> None:
logger.info("setting instance_id log export")
container_name = "base-config"
blob_name = "instance_id"
account_name = self.results["deploy"]["func-name"]["value"]
key = self.results["deploy"]["func-key"]["value"]
account_url = "https://%s.blob.core.windows.net" % account_name
client = BlobServiceClient(account_url, credential=key)
if container_name not in [x["name"] for x in client.list_containers()]:
client.create_container(container_name)
blob_client = client.get_blob_client(container_name, blob_name)
if blob_client.exists():
logger.debug("instance_id already exists")
instance_id = uuid.UUID(blob_client.download_blob().readall().decode())
else:
logger.debug("creating new instance_id")
instance_id = uuid.uuid4()
blob_client.upload_blob(str(instance_id))
logger.info("instance_id: %s", instance_id)
def get_container(blob_service_client: BlobServiceClient,
container_name: str) -> bool:
""" Get a container client
"""
logging.info('blob_service_client.list_containers()')
logging.info(list(blob_service_client.list_containers()))
try:
_ = blob_service_client.get_container_client(container_name)
container_client = blob_service_client.get_container_client(
container_name)
except:
container_client = blob_service_client.create_container(container_name)
return container_client
def upload_tools(self) -> None:
logger.info("uploading tools from %s", self.tools)
account_name = self.results["deploy"]["func-name"]["value"]
key = self.results["deploy"]["func-key"]["value"]
account_url = "https://%s.blob.core.windows.net" % account_name
client = BlobServiceClient(account_url, credential=key)
if "tools" not in [x["name"] for x in client.list_containers()]:
client.create_container("tools")
expiry = datetime.utcnow() + timedelta(minutes=30)
sas = generate_container_sas(
account_name,
"tools",
account_key=key,
permission=ContainerSasPermissions(read=True,
write=True,
delete=True,
list=True),
expiry=expiry,
)
url = "%s/%s?%s" % (account_url, "tools", sas)
subprocess.check_output([
self.azcopy,
"copy",
os.path.join(self.tools, "*"),
url,
"--overwrite=true",
"--recursive=true",
])
subprocess.check_output([
self.azcopy, "sync", self.tools, url, "--delete-destination",
"true"
])
def test_create_container_with_default_cpk_n_deny_override(
self, resource_group, location, storage_account,
storage_account_key):
# Arrange
bsc = BlobServiceClient(self.account_url(storage_account, "blob"),
credential=storage_account_key,
connection_data_block_size=1024,
max_single_put_size=1024,
min_large_block_upload_threshold=1024,
max_block_size=1024,
max_page_size=1024)
container_client = bsc.create_container(
'denyoverridecpkcontainer',
container_encryption_scope=
TEST_CONTAINER_ENCRYPTION_KEY_SCOPE_DENY_OVERRIDE)
container_props = container_client.get_container_properties()
self.assertEqual(
container_props.encryption_scope.default_encryption_scope,
TEST_CONTAINER_ENCRYPTION_KEY_SCOPE.default_encryption_scope)
self.assertEqual(
container_props.encryption_scope.prevent_encryption_scope_override,
True)
for container in bsc.list_containers(
name_starts_with='denyoverridecpkcontainer'):
self.assertEqual(
container_props.encryption_scope.default_encryption_scope,
TEST_CONTAINER_ENCRYPTION_KEY_SCOPE.default_encryption_scope)
self.assertEqual(
container_props.encryption_scope.
prevent_encryption_scope_override, True)
blob_client = container_client.get_blob_client("appendblob")
# It's not allowed to set encryption scope on the blob when the container denies encryption scope override.
with self.assertRaises(HttpResponseError):
blob_client.upload_blob(b'aaaa',
BlobType.AppendBlob,
encryption_scope=TEST_ENCRYPTION_KEY_SCOPE)
resp = blob_client.upload_blob(b'aaaa', BlobType.AppendBlob)
self.assertEqual(
resp['encryption_scope'],
TEST_CONTAINER_ENCRYPTION_KEY_SCOPE.default_encryption_scope)
container_client.delete_container()
def test_request_callback_signed_header(self, resource_group, location, storage_account, storage_account_key):
# Arrange
service = BlobServiceClient(self._account_url(storage_account.name), credential=storage_account_key)
name = self.get_resource_name('cont')
# Act
def callback(request):
if request.http_request.method == 'PUT':
request.http_request.headers['x-ms-meta-hello'] = 'world'
# Assert
try:
container = service.create_container(name, raw_request_hook=callback)
metadata = container.get_container_properties().metadata
self.assertEqual(metadata, {'hello': 'world'})
finally:
service.delete_container(name)
def main(mytimer: func.TimerRequest) -> None:
storageName = os.getenv("STORAGE_ACCOUNT_NAME")
containerName = os.getenv("BLOB_CONTAINER_NAME")
logging.info(f"params: {storageName}, {containerName}")
ts = datetime.datetime.now()
iso_ts = ts.isoformat()
date_str = ts.strftime("%Y%m%d")
credential = DefaultAzureCredential()
oauth_url = f"https://{storageName}.blob.core.windows.net"
bsc = BlobServiceClient(account_url=oauth_url, credential=credential)
containerClient = bsc.get_container_client(containerName)
tickers = ["BTC-USD", "MSFT"]
n = len(tickers)
for i in range(n):
block = ",\n"
ticker = tickers[i]
logging.info(f"storing {ticker}")
blobName = createBlobName(ticker, date_str)
logging.info(f"container: {containerName}, blob: {blobName}")
blobClient = containerClient.get_blob_client(blobName)
try:
if not blobClient.exists():
blobClient.create_append_blob()
block = ""
except ResourceNotFoundError:
logging.info(f"Creating container: {containerName}")
containerClient = bsc.create_container(containerName)
blobClient = containerClient.get_blob_client(blobName)
blobClient.create_append_blob()
block += json.dumps({"ts": iso_ts, "v": getCurrClose(ticker)})
blobClient.append_block(block)
if i<n-1: time.sleep(1)
def test_create_container_with_default_cpk_n(self, resource_group,
location, storage_account,
storage_account_key):
# Arrange
bsc = BlobServiceClient(self.account_url(storage_account, "blob"),
credential=storage_account_key,
connection_data_block_size=1024,
max_single_put_size=1024,
min_large_block_upload_threshold=1024,
max_block_size=1024,
max_page_size=1024)
container_client = bsc.create_container(
'cpkcontainer',
container_encryption_scope=TEST_CONTAINER_ENCRYPTION_KEY_SCOPE)
container_props = container_client.get_container_properties()
self.assertEqual(
container_props.encryption_scope.default_encryption_scope,
TEST_CONTAINER_ENCRYPTION_KEY_SCOPE.default_encryption_scope)
self.assertEqual(
container_props.encryption_scope.prevent_encryption_scope_override,
False)
for container in bsc.list_containers(name_starts_with='cpkcontainer'):
self.assertEqual(
container_props.encryption_scope.default_encryption_scope,
TEST_CONTAINER_ENCRYPTION_KEY_SCOPE.default_encryption_scope)
self.assertEqual(
container_props.encryption_scope.
prevent_encryption_scope_override, False)
blob_client = container_client.get_blob_client("appendblob")
# providing encryption scope when upload the blob
resp = blob_client.upload_blob(
b'aaaa',
BlobType.AppendBlob,
encryption_scope=TEST_ENCRYPTION_KEY_SCOPE)
# Use the provided encryption scope on the blob
self.assertEqual(resp['encryption_scope'], TEST_ENCRYPTION_KEY_SCOPE)
container_client.delete_container()
def create_writable_container_sas(self, account_name, account_key,
container_name, access_duration_hrs):
account_url = "https://{}.blob.core.windows.net".format(account_name)
blob_service_client = BlobServiceClient(account_url=account_url,
credential=account_key)
container_client = blob_service_client.create_container(container_name)
sas_permissions = ContainerSasPermissions(read=True,
write=True,
delete=False,
list=True)
expiration = datetime.utcnow() + timedelta(hours=access_duration_hrs)
sas_token = generate_container_sas(account_name,
container_name,
account_key=account_key,
permission=sas_permissions,
expiry=expiration)
return '{}/{}?{}'.format(account_url, container_name, sas_token)
class AzureBlobFileSystem(AbstractFileSystem):
"""
Access Azure Datalake Gen2 and Azure Storage if it were a file system using Multiprotocol Access
Parameters
----------
account_name: str
The storage account name. This is used to authenticate requests
signed with an account key and to construct the storage endpoint. It
is required unless a connection string is given, or if a custom
domain is used with anonymous authentication.
account_key: str
The storage account key. This is used for shared key authentication.
If any of account key, sas token or client_id is specified, anonymous access
will be used.
sas_token: str
A shared access signature token to use to authenticate requests
instead of the account key. If account key and sas token are both
specified, account key will be used to sign. If any of account key, sas token
or client_id are specified, anonymous access will be used.
request_session: Session
The session object to use for http requests.
connection_string: str
If specified, this will override all other parameters besides
request session. See
http://azure.microsoft.com/en-us/documentation/articles/storage-configure-connection-string/
for the connection string format.
socket_timeout: int
If specified, this will override the default socket timeout. The timeout specified is in
seconds.
See DEFAULT_SOCKET_TIMEOUT in _constants.py for the default value.
token_credential: TokenCredential
A token credential used to authenticate HTTPS requests. The token value
should be updated before its expiration.
blocksize: int
The block size to use for download/upload operations. Defaults to the value of
``BlockBlobService.MAX_BLOCK_SIZE``
client_id: str
Client ID to use when authenticating using an AD Service Principal client/secret.
client_secret: str
Client secret to use when authenticating using an AD Service Principal client/secret.
tenant_id: str
Tenant ID to use when authenticating using an AD Service Principal client/secret.
Examples
--------
>>> abfs = AzureBlobFileSystem(account_name="XXXX", account_key="XXXX", container_name="XXXX")
>>> abfs.ls('')
** Sharded Parquet & csv files can be read as: **
------------------------------------------
ddf = dd.read_csv('abfs://container_name/folder/*.csv', storage_options={
... 'account_name': ACCOUNT_NAME, 'account_key': ACCOUNT_KEY})
ddf = dd.read_parquet('abfs://container_name/folder.parquet', storage_options={
... 'account_name': ACCOUNT_NAME, 'account_key': ACCOUNT_KEY,})
"""
protocol = "abfs"
def __init__(
self,
account_name: str,
account_key: str = None,
connection_string: str = None,
credential: str = None,
sas_token: str = None,
request_session=None,
socket_timeout: int = None,
token_credential=None,
blocksize: int = create_configuration(storage_sdk="blob").max_block_size,
client_id: str = None,
client_secret: str = None,
tenant_id: str = None,
):
AbstractFileSystem.__init__(self)
self.account_name = account_name
self.account_key = account_key
self.connection_string = connection_string
self.credential = credential
self.sas_token = sas_token
self.request_session = request_session
self.socket_timeout = socket_timeout
self.token_credential = token_credential
self.blocksize = blocksize
self.client_id = client_id
self.client_secret = client_secret
self.tenant_id = tenant_id
if (
self.token_credential is None
and self.account_key is None
and self.sas_token is None
and self.client_id is not None
):
self.token_credential = self._get_token_from_service_principal()
self.do_connect()
@classmethod
def _strip_protocol(cls, path: str):
"""
Remove the protocol from the input path
Parameters
----------
path: str
Path to remove the protocol from
Returns
-------
str
Returns a path without the protocol
"""
logging.debug(f"_strip_protocol for {path}")
ops = infer_storage_options(path)
# we need to make sure that the path retains
# the format {host}/{path}
# here host is the container_name
if ops.get("host", None):
ops["path"] = ops["host"] + ops["path"]
ops["path"] = ops["path"].lstrip("/")
logging.debug(f"_strip_protocol({path}) = {ops}")
return ops["path"]
def _get_token_from_service_principal(self):
"""
Create a TokenCredential given a client_id, client_secret and tenant_id
Returns
-------
TokenCredential
"""
from azure.common.credentials import ServicePrincipalCredentials
from azure.storage.common import TokenCredential
sp_cred = ServicePrincipalCredentials(
client_id=self.client_id,
secret=self.client_secret,
tenant=self.tenant_id,
resource="https://storage.azure.com/",
)
token_cred = TokenCredential(sp_cred.token["access_token"])
return token_cred
def do_connect(self):
"""Connect to the BlobServiceClient, using user-specified connection details.
Tries credentials first, then connection string and finally account key
Raises
------
ValueError if none of the connection details are available
"""
self.account_url: str = f"https://{self.account_name}.blob.core.windows.net"
if self.credential is not None:
self.service_client = BlobServiceClient(
account_url=self.account_url, credential=self.credential
)
elif self.connection_string is not None:
self.service_client = BlobServiceClient.from_connection_string(
conn_str=self.connection_string
)
elif self.account_key is not None:
self.service_client = BlobServiceClient(
account_url=self.account_url, credential=self.account_key
)
else:
raise ValueError("unable to connect with provided params!!")
def split_path(self, path, delimiter="/", return_container: bool = False, **kwargs):
"""
Normalize ABFS path string into bucket and key.
Parameters
----------
path : string
Input path, like `abfs://my_container/path/to/file`
delimiter: string
Delimiter used to split the path
return_container: bool
Examples
--------
>>> split_path("abfs://my_container/path/to/file")
['my_container', 'path/to/file']
"""
if path in ["", delimiter]:
return "", ""
path = self._strip_protocol(path)
path = path.lstrip(delimiter)
if "/" not in path:
# this means path is the container_name
return path, ""
else:
return path.split(delimiter, 1)
# def _generate_blobs(self, *args, **kwargs):
# """Follow next_marker to get ALL results."""
# logging.debug("running _generate_blobs...")
# blobs = self.blob_fs.list_blobs(*args, **kwargs)
# yield from blobs
# while blobs.next_marker:
# logging.debug(f"following next_marker {blobs.next_marker}")
# kwargs["marker"] = blobs.next_marker
# blobs = self.blob_fs.list_blobs(*args, **kwargs)
# yield from blobs
# def _matches(
# self, container_name, path, as_directory=False, delimiter="/", **kwargs
# ):
# """check if the path returns an exact match"""
# path = path.rstrip(delimiter)
# gen = self.blob_fs.list_blob_names(
# container_name=container_name,
# prefix=path,
# delimiter=delimiter,
# num_results=None,
# )
# contents = list(gen)
# if not contents:
# return False
# if as_directory:
# return contents[0] == path + delimiter
# else:
# return contents[0] == path
def ls(
self,
path: str,
detail: bool = False,
invalidate_cache: bool = True,
delimiter: str = "/",
return_glob: bool = False,
**kwargs,
):
"""
Create a list of blob names from a blob container
Parameters
----------
path: str
Path to an Azure Blob with its container name
detail: bool
If False, return a list of blob names, else a list of dictionaries with blob details
invalidate_cache: bool
If True, do not use the cache
delimiter: str
Delimiter used to split paths
return_glob: bool
"""
logging.debug(f"abfs.ls() is searching for {path}")
container, path = self.split_path(path)
if (container in ["", delimiter]) and (path in ["", delimiter]):
# This is the case where only the containers are being returned
logging.info(
"Returning a list of containers in the azure blob storage account"
)
if detail:
contents = self.service_client.list_containers(include_metadata=True)
return self._details(contents)
else:
contents = self.service_client.list_containers()
return [f"{c.name}{delimiter}" for c in contents]
else:
if container not in ["", delimiter]:
# This is the case where the container name is passed
container_client = self.service_client.get_container_client(
container=container
)
blobs = container_client.walk_blobs(name_starts_with=path)
try:
blobs = [blob for blob in blobs]
except Exception:
raise FileNotFoundError
if len(blobs) > 1:
if return_glob:
return self._details(blobs, return_glob=True)
if detail:
return self._details(blobs)
else:
return [
f"{blob.container}{delimiter}{blob.name}" for blob in blobs
]
elif len(blobs) == 1:
if (blobs[0].name.rstrip(delimiter) == path) and not blobs[
0
].has_key( # NOQA
"blob_type"
):
path = blobs[0].name
blobs = container_client.walk_blobs(name_starts_with=path)
if return_glob:
return self._details(blobs, return_glob=True)
if detail:
return self._details(blobs)
else:
return [
f"{blob.container}{delimiter}{blob.name}"
for blob in blobs
]
elif isinstance(blobs[0], BlobPrefix):
if detail:
for blob_page in blobs:
return self._details(blob_page)
else:
outblobs = []
for blob_page in blobs:
for blob in blob_page:
outblobs.append(
f"{blob.container}{delimiter}{blob.name}"
)
return outblobs
elif blobs[0]["blob_type"] == "BlockBlob":
if detail:
return self._details(blobs)
else:
return [
f"{blob.container}{delimiter}{blob.name}"
for blob in blobs
]
elif isinstance(blobs[0], ItemPaged):
outblobs = []
for page in blobs:
for b in page:
outblobs.append(b)
else:
raise FileNotFoundError(
f"Unable to identify blobs in {path} for {blobs[0].name}"
)
elif len(blobs) == 0:
if return_glob or (path in ["", delimiter]):
return []
else:
raise FileNotFoundError
else:
raise FileNotFoundError
def _details(self, contents, delimiter="/", return_glob: bool = False, **kwargs):
"""
Return a list of dictionaries of specifying details about the contents
Parameters
----------
contents
delimiter: str
Delimiter used to separate containers and files
return_glob: bool
Returns
-------
List of dicts
Returns details about the contents, such as name, size and type
"""
pathlist = []
for c in contents:
data = {}
if c.has_key("container"): # NOQA
data["name"] = f"{c.container}{delimiter}{c.name}"
if c.has_key("size"): # NOQA
data["size"] = c.size
else:
data["size"] = 0
if data["size"] == 0:
data["type"] = "directory"
else:
data["type"] = "file"
else:
data["name"] = f"{c.name}{delimiter}"
data["size"] = 0
data["type"] = "directory"
if return_glob:
data["name"] = data["name"].rstrip("/")
pathlist.append(data)
return pathlist
def walk(self, path: str, maxdepth=None, **kwargs):
""" Return all files belows path
List all files, recursing into subdirectories; output is iterator-style,
like ``os.walk()``. For a simple list of files, ``find()`` is available.
Note that the "files" outputted will include anything that is not
a directory, such as links.
Parameters
----------
path: str
Root to recurse into
maxdepth: int
Maximum recursion depth. None means limitless, but not recommended
on link-based file-systems.
**kwargs are passed to ``ls``
"""
path = self._strip_protocol(path)
full_dirs = {}
dirs = {}
files = {}
detail = kwargs.pop("detail", False)
try:
listing = self.ls(path, detail=True, return_glob=True, **kwargs)
except (FileNotFoundError, IOError):
return [], [], []
for info in listing:
# each info name must be at least [path]/part , but here
# we check also for names like [path]/part/
pathname = info["name"].rstrip("/")
name = pathname.rsplit("/", 1)[-1]
if info["type"] == "directory" and pathname != path:
# do not include "self" path
full_dirs[pathname] = info
dirs[name] = info
elif pathname == path:
# file-like with same name as give path
files[""] = info
else:
files[name] = info
if detail:
yield path, dirs, files
else:
yield path, list(dirs), list(files)
if maxdepth is not None:
maxdepth -= 1
if maxdepth < 1:
return
for d in full_dirs:
yield from self.walk(d, maxdepth=maxdepth, detail=detail, **kwargs)
def mkdir(self, path, delimiter="/", exists_ok=False, **kwargs):
"""
Create directory entry at path
Parameters
----------
path: str
The path to create
delimiter: str
Delimiter to use when splitting the path
exists_ok: bool
If True, raise an exception if the directory already exists. Defaults to False
"""
container_name, path = self.split_path(path, delimiter=delimiter)
if not exists_ok:
if (container_name not in self.ls("")) and (not path):
# create new container
self.service_client.create_container(name=container_name)
elif (
container_name
in [container_path.split("/")[0] for container_path in self.ls("")]
) and path:
## attempt to create prefix
container_client = self.service_client.get_container_client(
container=container_name
)
container_client.upload_blob(name=path, data="")
else:
## everything else
raise RuntimeError(f"Cannot create {container_name}{delimiter}{path}.")
else:
if container_name in self.ls("") and path:
container_client = self.service_client.get_container_client(
container=container_name
)
container_client.upload_blob(name=path, data="")
def rmdir(self, path: str, delimiter="/", **kwargs):
"""
Remove a directory, if empty
Parameters
----------
path: str
Path of directory to remove
delimiter: str
Delimiter to use when splitting the path
"""
container_name, path = self.split_path(path, delimiter=delimiter)
if (container_name + delimiter in self.ls("")) and (not path):
# delete container
self.service_client.delete_container(container_name)
def _rm(self, path, delimiter="/", **kwargs):
"""
Delete a given file
Parameters
----------
path: str
Path to file to delete
delimiter: str
Delimiter to use when splitting the path
"""
if self.isfile(path):
container_name, path = self.split_path(path, delimiter=delimiter)
container_client = self.service_client.get_container_client(
container=container_name
)
logging.debug(f"Delete blob {path} in {container_name}")
container_client.delete_blob(path)
elif self.isdir(path):
container_name, path = self.split_path(path, delimiter=delimiter)
container_client = self.service_client.get_container_client(
container=container_name
)
if (container_name + delimiter in self.ls("")) and (not path):
logging.debug(f"Delete container {container_name}")
container_client.delete_container(container_name)
else:
raise RuntimeError(f"cannot delete {path}")
def _open(
self,
path: str,
mode: str = "rb",
block_size: int = None,
autocommit: bool = True,
cache_options=None,
**kwargs,
):
"""Open a file on the datalake, or a block blob
Parameters
----------
path: str
Path to file to open
mode: str
What mode to open the file in - defaults to "rb"
block_size: int
Size per block for multi-part downloads.
autocommit: bool
Whether or not to write to the destination directly
cache_type: str
One of "readahead", "none", "mmap", "bytes", defaults to "readahead"
Caching policy in read mode.
See the definitions here:
https://filesystem-spec.readthedocs.io/en/latest/api.html#readbuffering
"""
logging.debug(f"_open: {path}")
return AzureBlobFile(
fs=self,
path=path,
mode=mode,
block_size=block_size or self.blocksize,
autocommit=autocommit,
cache_options=cache_options,
**kwargs,
)
class StorageBlockBlobTest(StorageTestCase):
def setUp(self):
super(StorageBlockBlobTest, self).setUp()
url = self._get_account_url()
credential = self._get_shared_key_credential()
# test chunking functionality by reducing the size of each chunk,
# otherwise the tests would take too long to execute
self.bsc = BlobServiceClient(url,
credential=credential,
connection_data_block_size=4 * 1024,
max_single_put_size=32 * 1024,
max_block_size=4 * 1024)
self.config = self.bsc._config
self.container_name = self.get_resource_name('utcontainer')
# create source blob to be copied from
self.source_blob_name = self.get_resource_name('srcblob')
self.source_blob_data = self.get_random_bytes(SOURCE_BLOB_SIZE)
blob = self.bsc.get_blob_client(self.container_name,
self.source_blob_name)
if not self.is_playback():
self.bsc.create_container(self.container_name)
blob.upload_blob(self.source_blob_data)
# generate a SAS so that it is accessible with a URL
sas_token = blob.generate_shared_access_signature(
permission=BlobPermissions.READ,
expiry=datetime.utcnow() + timedelta(hours=1),
)
self.source_blob_url = BlobClient(blob.url, credential=sas_token).url
def tearDown(self):
if not self.is_playback():
try:
self.bsc.delete_container(self.container_name)
except HttpResponseError:
pass
return super(StorageBlockBlobTest, self).tearDown()
@record
def test_put_block_from_url_and_commit(self):
# Arrange
dest_blob_name = self.get_resource_name('destblob')
dest_blob = self.bsc.get_blob_client(self.container_name,
dest_blob_name)
# Act part 1: make put block from url calls
dest_blob.stage_block_from_url(block_id=1,
source_url=self.source_blob_url,
source_offset=0,
source_length=4 * 1024 - 1)
dest_blob.stage_block_from_url(block_id=2,
source_url=self.source_blob_url,
source_offset=4 * 1024,
source_length=8 * 1024)
# Assert blocks
committed, uncommitted = dest_blob.get_block_list('all')
self.assertEqual(len(uncommitted), 2)
self.assertEqual(len(committed), 0)
# Act part 2: commit the blocks
dest_blob.commit_block_list(['1', '2'])
# Assert destination blob has right content
content = dest_blob.download_blob().content_as_bytes()
self.assertEqual(content, self.source_blob_data)
@record
def test_put_block_from_url_and_validate_content_md5(self):
# Arrange
dest_blob_name = self.get_resource_name('destblob')
dest_blob = self.bsc.get_blob_client(self.container_name,
dest_blob_name)
src_md5 = StorageContentValidation.get_content_md5(
self.source_blob_data)
# Act part 1: put block from url with md5 validation
dest_blob.stage_block_from_url(block_id=1,
source_url=self.source_blob_url,
source_content_md5=src_md5,
source_offset=0,
source_length=8 * 1024)
# Assert block was staged
committed, uncommitted = dest_blob.get_block_list('all')
self.assertEqual(len(uncommitted), 1)
self.assertEqual(len(committed), 0)
# Act part 2: put block from url with wrong md5
fake_md5 = StorageContentValidation.get_content_md5(b"POTATO")
with self.assertRaises(HttpResponseError) as error:
dest_blob.stage_block_from_url(block_id=2,
source_url=self.source_blob_url,
source_content_md5=fake_md5,
source_offset=0,
source_length=8 * 1024)
self.assertEqual(error.exception.error_code,
StorageErrorCode.md5_mismatch)
# Assert block was not staged
committed, uncommitted = dest_blob.get_block_list('all')
self.assertEqual(len(uncommitted), 1)
self.assertEqual(len(committed), 0)
@record
def test_copy_blob_sync(self):
# Arrange
dest_blob_name = self.get_resource_name('destblob')
dest_blob = self.bsc.get_blob_client(self.container_name,
dest_blob_name)
# Act
copy_props = dest_blob.copy_blob_from_url(self.source_blob_url,
requires_sync=True)
# Assert
self.assertIsNotNone(copy_props)
self.assertIsNotNone(copy_props.copy_id())
self.assertEqual('success', copy_props.status())
# Verify content
content = dest_blob.download_blob().content_as_bytes()
self.assertEqual(self.source_blob_data, content)
class StorageLargeBlockBlobTest(StorageTestCase):
def _setup(self, storage_account, key):
# test chunking functionality by reducing the threshold
# for chunking and the size of each chunk, otherwise
# the tests would take too long to execute
self.bsc = BlobServiceClient(self.account_url(storage_account, "blob"),
credential=key,
max_single_put_size=32 * 1024,
max_block_size=2 * 1024 * 1024,
min_large_block_upload_threshold=1 *
1024 * 1024)
self.config = self.bsc._config
self.container_name = self.get_resource_name('utcontainer')
if self.is_live:
self.bsc.create_container(self.container_name)
def _teardown(self, file_name):
if path.isfile(file_name):
try:
remove(file_name)
except:
pass
# --Helpers-----------------------------------------------------------------
def _get_blob_reference(self):
return self.get_resource_name(TEST_BLOB_PREFIX)
def _create_blob(self):
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
blob.upload_blob(b'')
return blob
def assertBlobEqual(self, container_name, blob_name, expected_data):
blob = self.bsc.get_blob_client(container_name, blob_name)
actual_data = blob.download_blob()
self.assertEqual(b"".join(list(actual_data.chunks())), expected_data)
# --Test cases for block blobs --------------------------------------------
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
def test_put_block_bytes_large(self, resource_group, location,
storage_account, storage_account_key):
self._setup(storage_account, storage_account_key)
blob = self._create_blob()
# Act
for i in range(5):
resp = blob.stage_block('block {0}'.format(i).encode('utf-8'),
urandom(LARGE_BLOCK_SIZE))
self.assertIsNotNone(resp)
assert 'content_md5' in resp
assert 'content_crc64' in resp
assert 'request_id' in resp
# Assert
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
def test_put_block_bytes_large_with_md5(self, resource_group, location,
storage_account,
storage_account_key):
self._setup(storage_account, storage_account_key)
blob = self._create_blob()
# Act
for i in range(5):
resp = blob.stage_block('block {0}'.format(i).encode('utf-8'),
urandom(LARGE_BLOCK_SIZE),
validate_content=True)
self.assertIsNotNone(resp)
assert 'content_md5' in resp
assert 'content_crc64' in resp
assert 'request_id' in resp
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
def test_put_block_stream_large(self, resource_group, location,
storage_account, storage_account_key):
self._setup(storage_account, storage_account_key)
blob = self._create_blob()
# Act
for i in range(5):
stream = BytesIO(bytearray(LARGE_BLOCK_SIZE))
resp = resp = blob.stage_block(
'block {0}'.format(i).encode('utf-8'),
stream,
length=LARGE_BLOCK_SIZE)
self.assertIsNotNone(resp)
assert 'content_md5' in resp
assert 'content_crc64' in resp
assert 'request_id' in resp
# Assert
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
def test_put_block_stream_large_with_md5(self, resource_group, location,
storage_account,
storage_account_key):
self._setup(storage_account, storage_account_key)
blob = self._create_blob()
# Act
for i in range(5):
stream = BytesIO(bytearray(LARGE_BLOCK_SIZE))
resp = resp = blob.stage_block(
'block {0}'.format(i).encode('utf-8'),
stream,
length=LARGE_BLOCK_SIZE,
validate_content=True)
self.assertIsNotNone(resp)
assert 'content_md5' in resp
assert 'content_crc64' in resp
assert 'request_id' in resp
# Assert
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
def test_create_large_blob_from_path(self, resource_group, location,
storage_account, storage_account_key):
# parallel tests introduce random order of requests, can only run live
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = bytearray(urandom(LARGE_BLOB_SIZE))
FILE_PATH = 'large_blob_from_path.temp.{}.dat'.format(str(
uuid.uuid4()))
with open(FILE_PATH, 'wb') as stream:
stream.write(data)
# Act
with open(FILE_PATH, 'rb') as stream:
blob.upload_blob(stream, max_concurrency=2)
# Assert
self.assertBlobEqual(self.container_name, blob_name, data)
self._teardown(FILE_PATH)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
def test_create_large_blob_from_path_with_md5(self, resource_group,
location, storage_account,
storage_account_key):
# parallel tests introduce random order of requests, can only run live
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = bytearray(urandom(LARGE_BLOB_SIZE))
FILE_PATH = "blob_from_path_with_md5.temp.dat"
with open(FILE_PATH, 'wb') as stream:
stream.write(data)
# Act
with open(FILE_PATH, 'rb') as stream:
blob.upload_blob(stream, validate_content=True, max_concurrency=2)
# Assert
self.assertBlobEqual(self.container_name, blob_name, data)
self._teardown(FILE_PATH)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
def test_create_large_blob_from_path_non_parallel(self, resource_group,
location,
storage_account,
storage_account_key):
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = bytearray(self.get_random_bytes(100))
FILE_PATH = "blob_from_path_non_parallel.temp.dat"
with open(FILE_PATH, 'wb') as stream:
stream.write(data)
# Act
with open(FILE_PATH, 'rb') as stream:
blob.upload_blob(stream, max_concurrency=1)
# Assert
self.assertBlobEqual(self.container_name, blob_name, data)
self._teardown(FILE_PATH)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
def test_create_large_blob_from_path_with_progress(self, resource_group,
location,
storage_account,
storage_account_key):
# parallel tests introduce random order of requests, can only run live
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = bytearray(urandom(LARGE_BLOB_SIZE))
FILE_PATH = "blob_from_path_with_progress.temp.dat"
with open(FILE_PATH, 'wb') as stream:
stream.write(data)
# Act
progress = []
def callback(response):
current = response.context['upload_stream_current']
total = response.context['data_stream_total']
if current is not None:
progress.append((current, total))
with open(FILE_PATH, 'rb') as stream:
blob.upload_blob(stream,
max_concurrency=2,
raw_response_hook=callback)
# Assert
self.assertBlobEqual(self.container_name, blob_name, data)
self.assert_upload_progress(len(data), self.config.max_block_size,
progress)
self._teardown(FILE_PATH)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
def test_create_large_blob_from_path_with_properties(
self, resource_group, location, storage_account,
storage_account_key):
# parallel tests introduce random order of requests, can only run live
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = bytearray(urandom(LARGE_BLOB_SIZE))
FILE_PATH = 'blob_from_path_with_properties.temp.{}.dat'.format(
str(uuid.uuid4()))
with open(FILE_PATH, 'wb') as stream:
stream.write(data)
# Act
content_settings = ContentSettings(content_type='image/png',
content_language='spanish')
with open(FILE_PATH, 'rb') as stream:
blob.upload_blob(stream,
content_settings=content_settings,
max_concurrency=2)
# Assert
self.assertBlobEqual(self.container_name, blob_name, data)
properties = blob.get_blob_properties()
self.assertEqual(properties.content_settings.content_type,
content_settings.content_type)
self.assertEqual(properties.content_settings.content_language,
content_settings.content_language)
self._teardown(FILE_PATH)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
def test_create_large_blob_from_stream_chunked_upload(
self, resource_group, location, storage_account,
storage_account_key):
# parallel tests introduce random order of requests, can only run live
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = bytearray(urandom(LARGE_BLOB_SIZE))
FILE_PATH = 'blob_from_stream_chunked_upload.temp.{}.dat'.format(
str(uuid.uuid4()))
with open(FILE_PATH, 'wb') as stream:
stream.write(data)
# Act
with open(FILE_PATH, 'rb') as stream:
blob.upload_blob(stream, max_concurrency=2)
# Assert
self.assertBlobEqual(self.container_name, blob_name, data)
self._teardown(FILE_PATH)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
def test_creat_lrgblob_frm_stream_w_progress_chnkd_upload(
self, resource_group, location, storage_account,
storage_account_key):
# parallel tests introduce random order of requests, can only run live
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = bytearray(urandom(LARGE_BLOB_SIZE))
FILE_PATH = 'stream_w_progress_chnkd_upload.temp.{}.dat'.format(
str(uuid.uuid4()))
with open(FILE_PATH, 'wb') as stream:
stream.write(data)
# Act
progress = []
def callback(response):
current = response.context['upload_stream_current']
total = response.context['data_stream_total']
if current is not None:
progress.append((current, total))
with open(FILE_PATH, 'rb') as stream:
blob.upload_blob(stream,
max_concurrency=2,
raw_response_hook=callback)
# Assert
self.assertBlobEqual(self.container_name, blob_name, data)
self.assert_upload_progress(len(data), self.config.max_block_size,
progress)
self._teardown(FILE_PATH)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
def test_create_large_blob_from_stream_chunked_upload_with_count(
self, resource_group, location, storage_account,
storage_account_key):
# parallel tests introduce random order of requests, can only run live
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = bytearray(urandom(LARGE_BLOB_SIZE))
FILE_PATH = 'chunked_upload_with_count.temp.{}.dat'.format(
str(uuid.uuid4()))
with open(FILE_PATH, 'wb') as stream:
stream.write(data)
# Act
blob_size = len(data) - 301
with open(FILE_PATH, 'rb') as stream:
blob.upload_blob(stream, length=blob_size, max_concurrency=2)
# Assert
self.assertBlobEqual(self.container_name, blob_name, data[:blob_size])
self._teardown(FILE_PATH)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
def test_creat_lrgblob_frm_strm_chnkd_uplod_w_count_n_props(
self, resource_group, location, storage_account,
storage_account_key):
# parallel tests introduce random order of requests, can only run live
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = bytearray(urandom(LARGE_BLOB_SIZE))
FILE_PATH = 'plod_w_count_n_props.temp.{}.dat'.format(str(
uuid.uuid4()))
with open(FILE_PATH, 'wb') as stream:
stream.write(data)
# Act
content_settings = ContentSettings(content_type='image/png',
content_language='spanish')
blob_size = len(data) - 301
with open(FILE_PATH, 'rb') as stream:
blob.upload_blob(stream,
length=blob_size,
content_settings=content_settings,
max_concurrency=2)
# Assert
self.assertBlobEqual(self.container_name, blob_name, data[:blob_size])
properties = blob.get_blob_properties()
self.assertEqual(properties.content_settings.content_type,
content_settings.content_type)
self.assertEqual(properties.content_settings.content_language,
content_settings.content_language)
self._teardown(FILE_PATH)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
def test_creat_lrg_blob_frm_stream_chnked_upload_w_props(
self, resource_group, location, storage_account,
storage_account_key):
# parallel tests introduce random order of requests, can only run live
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = bytearray(urandom(LARGE_BLOB_SIZE))
FILE_PATH = 'creat_lrg_blob.temp.{}.dat'.format(str(uuid.uuid4()))
with open(FILE_PATH, 'wb') as stream:
stream.write(data)
# Act
content_settings = ContentSettings(content_type='image/png',
content_language='spanish')
with open(FILE_PATH, 'rb') as stream:
blob.upload_blob(stream,
content_settings=content_settings,
max_concurrency=2)
# Assert
self.assertBlobEqual(self.container_name, blob_name, data)
properties = blob.get_blob_properties()
self.assertEqual(properties.content_settings.content_type,
content_settings.content_type)
self.assertEqual(properties.content_settings.content_language,
content_settings.content_language)
self._teardown(FILE_PATH)
class StorageBlobTagsTest(StorageTestCase):
def _setup(self, storage_account, key):
self.bsc = BlobServiceClient(self.account_url(storage_account, "blob"), credential=key)
self.container_name = self.get_resource_name("container")
if self.is_live:
container = self.bsc.get_container_client(self.container_name)
try:
container.create_container(timeout=5)
except ResourceExistsError:
pass
self.byte_data = self.get_random_bytes(1024)
def _teardown(self, FILE_PATH):
if os.path.isfile(FILE_PATH):
try:
os.remove(FILE_PATH)
except:
pass
#--Helpers-----------------------------------------------------------------
def _get_blob_reference(self):
return self.get_resource_name(TEST_BLOB_PREFIX)
def _create_block_blob(self, tags=None, container_name=None, blob_name=None):
blob_name = blob_name or self._get_blob_reference()
blob_client = self.bsc.get_blob_client(container_name or self.container_name, blob_name)
resp = blob_client.upload_blob(self.byte_data, length=len(self.byte_data), overwrite=True, tags=tags)
return blob_client, resp
def _create_empty_block_blob(self, tags=None):
blob_name = self._get_blob_reference()
blob_client = self.bsc.get_blob_client(self.container_name, blob_name)
resp = blob_client.upload_blob(b'', length=0, overwrite=True, tags=tags)
return blob_client, resp
def _create_append_blob(self, tags=None):
blob_name = self._get_blob_reference()
blob_client = self.bsc.get_blob_client(self.container_name, blob_name)
resp = blob_client.create_append_blob(tags=tags)
return blob_client, resp
def _create_page_blob(self, tags=None):
blob_name = self._get_blob_reference()
blob_client = self.bsc.get_blob_client(self.container_name, blob_name)
resp = blob_client.create_page_blob(tags=tags, size=512)
return blob_client, resp
def _create_container(self, prefix="container"):
container_name = self.get_resource_name(prefix)
try:
self.bsc.create_container(container_name)
except:
pass
return container_name
#-- test cases for blob tags ----------------------------------------------
@GlobalResourceGroupPreparer()
@StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage')
def test_set_blob_tags(self, resource_group, location, storage_account, storage_account_key):
self._setup(storage_account, storage_account_key)
blob_client, _ = self._create_block_blob()
# Act
blob_tags = {"tag1": "firsttag", "tag2": "secondtag", "tag3": "thirdtag"}
resp = blob_client.set_blob_tags(blob_tags)
# Assert
self.assertIsNotNone(resp)
@pytest.mark.playback_test_only
@GlobalStorageAccountPreparer()
def test_set_blob_tags_for_a_version(self, resource_group, location, storage_account, storage_account_key):
self._setup(storage_account, storage_account_key)
# use this version to set tag
blob_client, resp = self._create_block_blob()
self._create_block_blob()
# TODO: enable versionid for this account and test set tag for a version
# Act
tags = {"tag1": "firsttag", "tag2": "secondtag", "tag3": "thirdtag"}
resp = blob_client.set_blob_tags(tags, version_id=resp['version_id'])
# Assert
self.assertIsNotNone(resp)
@GlobalResourceGroupPreparer()
@StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage')
def test_get_blob_tags(self, resource_group, location, storage_account, storage_account_key):
self._setup(storage_account, storage_account_key)
blob_client, resp = self._create_block_blob()
# Act
tags = {"tag1": "firsttag", "tag2": "secondtag", "tag3": "thirdtag"}
blob_client.set_blob_tags(tags)
resp = blob_client.get_blob_tags()
# Assert
self.assertIsNotNone(resp)
self.assertEqual(len(resp), 3)
for key, value in resp.items():
self.assertEqual(tags[key], value)
@GlobalResourceGroupPreparer()
@StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage')
def test_get_blob_tags_for_a_snapshot(self, resource_group, location, storage_account, storage_account_key):
self._setup(storage_account, storage_account_key)
tags = {"+-./:=_ ": "firsttag", "tag2": "+-./:=_", "+-./:=_1": "+-./:=_"}
blob_client, resp = self._create_block_blob(tags=tags)
snapshot = blob_client.create_snapshot()
snapshot_client = self.bsc.get_blob_client(self.container_name, blob_client.blob_name, snapshot=snapshot)
resp = snapshot_client.get_blob_tags()
# Assert
self.assertIsNotNone(resp)
self.assertEqual(len(resp), 3)
for key, value in resp.items():
self.assertEqual(tags[key], value)
@GlobalResourceGroupPreparer()
@StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage')
def test_upload_block_blob_with_tags(self, resource_group, location, storage_account, storage_account_key):
self._setup(storage_account, storage_account_key)
tags = {"tag1": "firsttag", "tag2": "secondtag", "tag3": "thirdtag"}
blob_client, resp = self._create_block_blob(tags=tags)
resp = blob_client.get_blob_tags()
# Assert
self.assertIsNotNone(resp)
self.assertEqual(len(resp), 3)
@GlobalResourceGroupPreparer()
@StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage')
def test_get_blob_properties_returns_tags_num(self, resource_group, location, storage_account, storage_account_key):
self._setup(storage_account, storage_account_key)
tags = {"tag1": "firsttag", "tag2": "secondtag", "tag3": "thirdtag"}
blob_client, resp = self._create_block_blob(tags=tags)
resp = blob_client.get_blob_properties()
downloaded = blob_client.download_blob()
# Assert
self.assertIsNotNone(resp)
self.assertEqual(resp.tag_count, len(tags))
self.assertEqual(downloaded.properties.tag_count, len(tags))
@GlobalResourceGroupPreparer()
@StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage')
def test_create_append_blob_with_tags(self, resource_group, location, storage_account, storage_account_key):
self._setup(storage_account, storage_account_key)
tags = {"+-./:=_ ": "firsttag", "tag2": "+-./:=_", "+-./:=_1": "+-./:=_"}
blob_client, resp = self._create_append_blob(tags=tags)
resp = blob_client.get_blob_tags()
# Assert
self.assertIsNotNone(resp)
self.assertEqual(len(resp), 3)
@GlobalResourceGroupPreparer()
@StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage')
def test_create_page_blob_with_tags(self, resource_group, location, storage_account, storage_account_key):
self._setup(storage_account, storage_account_key)
tags = {"tag1": "firsttag", "tag2": "secondtag", "tag3": "thirdtag"}
blob_client, resp = self._create_page_blob(tags=tags)
resp = blob_client.get_blob_tags()
# Assert
self.assertIsNotNone(resp)
self.assertEqual(len(resp), 3)
@GlobalResourceGroupPreparer()
@StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage')
def test_commit_block_list_with_tags(self, resource_group, location, storage_account, storage_account_key):
self._setup(storage_account, storage_account_key)
tags = {"tag1": "firsttag", "tag2": "secondtag", "tag3": "thirdtag"}
blob_client, resp = self._create_empty_block_blob(tags={'condition tag': 'test tag'})
blob_client.stage_block('1', b'AAA')
blob_client.stage_block('2', b'BBB')
blob_client.stage_block('3', b'CCC')
# Act
block_list = [BlobBlock(block_id='1'), BlobBlock(block_id='2'), BlobBlock(block_id='3')]
with self.assertRaises(ResourceModifiedError):
blob_client.commit_block_list(block_list, tags=tags, if_tags_match_condition="\"condition tag\"='wrong tag'")
blob_client.commit_block_list(block_list, tags=tags, if_tags_match_condition="\"condition tag\"='test tag'")
resp = blob_client.get_blob_tags()
# Assert
self.assertIsNotNone(resp)
self.assertEqual(len(resp), len(tags))
@GlobalResourceGroupPreparer()
@StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage')
def test_start_copy_from_url_with_tags(self, resource_group, location, storage_account, storage_account_key):
self._setup(storage_account, storage_account_key)
tags = {"tag1": "firsttag", "tag2": "secondtag", "tag3": "thirdtag"}
blob_client, resp = self._create_block_blob()
# Act
sourceblob = '{0}/{1}/{2}'.format(
self.account_url(storage_account, "blob"), self.container_name, blob_client.blob_name)
copyblob = self.bsc.get_blob_client(self.container_name, 'blob1copy')
copy = copyblob.start_copy_from_url(sourceblob, tags=tags)
# Assert
self.assertIsNotNone(copy)
self.assertEqual(copy['copy_status'], 'success')
self.assertFalse(isinstance(copy['copy_status'], Enum))
self.assertIsNotNone(copy['copy_id'])
copy_content = copyblob.download_blob().readall()
self.assertEqual(copy_content, self.byte_data)
resp = copyblob.get_blob_tags()
# Assert
self.assertIsNotNone(resp)
self.assertEqual(len(resp), len(tags))
@GlobalResourceGroupPreparer()
@StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage')
def test_list_blobs_returns_tags(self, resource_group, location, storage_account, storage_account_key):
self._setup(storage_account, storage_account_key)
tags = {"tag1": "firsttag", "tag2": "secondtag", "tag3": "thirdtag"}
self._create_block_blob(tags=tags)
container = self.bsc.get_container_client(self.container_name)
blob_list = container.list_blobs(include="tags")
#Assert
for blob in blob_list:
self.assertEqual(blob.tag_count, len(tags))
for key, value in blob.tags.items():
self.assertEqual(tags[key], value)
@GlobalResourceGroupPreparer()
@StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage')
def test_filter_blobs(self, resource_group, location, storage_account, storage_account_key):
self._setup(storage_account, storage_account_key)
container_name1 = self._create_container(prefix="container1")
container_name2 = self._create_container(prefix="container2")
container_name3 = self._create_container(prefix="container3")
tags = {"tag1": "firsttag", "tag2": "secondtag", "tag3": "thirdtag"}
self._create_block_blob(tags=tags, blob_name="blob1")
self._create_block_blob(tags=tags, blob_name="blob2", container_name=container_name1)
self._create_block_blob(tags=tags, blob_name="blob3", container_name=container_name2)
self._create_block_blob(tags=tags, blob_name="blob4", container_name=container_name3)
if self.is_live:
sleep(10)
where = "tag1='firsttag'"
blob_list = self.bsc.find_blobs_by_tags(filter_expression=where, results_per_page=2).by_page()
first_page = next(blob_list)
items_on_page1 = list(first_page)
second_page = next(blob_list)
items_on_page2 = list(second_page)
self.assertEqual(2, len(items_on_page1))
self.assertEqual(2, len(items_on_page2))
@pytest.mark.live_test_only
@GlobalResourceGroupPreparer()
@StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage')
def test_filter_blobs_using_account_sas(self, resource_group, location, storage_account, storage_account_key):
token = generate_account_sas(
storage_account.name,
storage_account_key,
ResourceTypes(service=True, container=True, object=True),
AccountSasPermissions(write=True, list=True, read=True, delete_previous_version=True, tag=True,
filter_by_tags=True),
datetime.utcnow() + timedelta(hours=1),
)
self._setup(storage_account, token)
tags = {"year": '1000', "tag2": "secondtag", "tag3": "thirdtag", "habitat_type": 'Shallow Lowland Billabongs'}
blob_client, _ = self._create_block_blob(tags=tags, container_name=self.container_name)
blob_client.set_blob_tags(tags=tags)
tags_on_blob = blob_client.get_blob_tags()
self.assertEqual(len(tags_on_blob), len(tags))
if self.is_live:
sleep(10)
# To filter in a specific container use:
# where = "@container='{}' and tag1='1000' and tag2 = 'secondtag'".format(container_name1)
where = "\"year\"='1000' and tag2 = 'secondtag' and tag3='thirdtag'"
blob_list = self.bsc.find_blobs_by_tags(filter_expression=where, results_per_page=2).by_page()
first_page = next(blob_list)
items_on_page1 = list(first_page)
self.assertEqual(1, len(items_on_page1))
@pytest.mark.live_test_only
@GlobalResourceGroupPreparer()
@StorageAccountPreparer(random_name_enabled=True, location="canadacentral", name_prefix='pytagstorage')
def test_set_blob_tags_using_blob_sas(self, resource_group, location, storage_account, storage_account_key):
token = generate_account_sas(
storage_account.name,
storage_account_key,
ResourceTypes(service=True, container=True, object=True),
AccountSasPermissions(write=True, list=True, read=True, delete_previous_version=True, tag=True,
filter_by_tags=True),
datetime.utcnow() + timedelta(hours=1),
)
self._setup(storage_account, token)
tags = {"year": '1000', "tag2": "secondtag", "tag3": "thirdtag", "habitat_type": 'Shallow Lowland Billabongs'}
blob_client, _ = self._create_block_blob(tags=tags, container_name=self.container_name)
token1 = generate_blob_sas(
storage_account.name,
self.container_name,
blob_client.blob_name,
account_key=storage_account_key,
permission=BlobSasPermissions(delete_previous_version=True, tag=True),
expiry=datetime.utcnow() + timedelta(hours=1),
)
blob_client=BlobClient.from_blob_url(blob_client.url, token1)
blob_client.set_blob_tags(tags=tags)
tags_on_blob = blob_client.get_blob_tags()
self.assertEqual(len(tags_on_blob), len(tags))
if self.is_live:
sleep(10)
# To filter in a specific container use:
# where = "@container='{}' and tag1='1000' and tag2 = 'secondtag'".format(container_name1)
where = "\"year\"='1000' and tag2 = 'secondtag' and tag3='thirdtag'"
blob_list = self.bsc.find_blobs_by_tags(filter_expression=where, results_per_page=2).by_page()
first_page = next(blob_list)
items_on_page1 = list(first_page)
self.assertEqual(1, len(items_on_page1))
if delete_extra_containers:
# c = extra_containers[0]
for c in extra_containers:
print('Delete container {} from storage account {}?'.format(
c, target_blob_service_client.account_name))
if confirm():
target_blob_service_client.delete_container(c)
#%% Create missing containers
# c = missing_containers[0]
for c in missing_containers:
print('Creating container {} in storage account {}'.format(
c, target_blob_service_client.account_name))
target_blob_service_client.create_container(c)
#%% Generate azcopy commands
azcopy_commands = []
# c = source_container_names[0]
for c in source_container_names:
cmd = 'azcopy sync "'
cmd += source_account_url_blob + '/' + c + '/' + source_sas_token
cmd += '" "'
cmd += target_account_url_blob + '/' + c + '/' + target_sas_token
cmd += '" --delete-destination=True --log-level=NONE'
azcopy_commands.append(cmd)
class StorageBlockBlobTest(StorageTestCase):
def _setup(self, storage_account, key):
self.bsc = BlobServiceClient(self.account_url(storage_account, "blob"),
credential=key,
connection_data_block_size=4 * 1024,
max_single_put_size=32 * 1024,
max_block_size=4 * 1024)
self.config = self.bsc._config
self.container_name = self.get_resource_name('utcontainer')
# create source blob to be copied from
self.source_blob_name = self.get_resource_name('srcblob')
self.source_blob_data = self.get_random_bytes(SOURCE_BLOB_SIZE)
blob = self.bsc.get_blob_client(self.container_name,
self.source_blob_name)
if self.is_live:
self.bsc.create_container(self.container_name)
blob.upload_blob(self.source_blob_data)
# generate a SAS so that it is accessible with a URL
sas_token = generate_blob_sas(
blob.account_name,
blob.container_name,
blob.blob_name,
snapshot=blob.snapshot,
account_key=blob.credential.account_key,
permission=BlobSasPermissions(read=True),
expiry=datetime.utcnow() + timedelta(hours=1),
)
self.source_blob_url = BlobClient.from_blob_url(
blob.url, credential=sas_token).url
@GlobalStorageAccountPreparer()
def test_put_block_from_url_and_commit(self, resource_group, location,
storage_account,
storage_account_key):
self._setup(storage_account, storage_account_key)
dest_blob_name = self.get_resource_name('destblob')
dest_blob = self.bsc.get_blob_client(self.container_name,
dest_blob_name)
# Act part 1: make put block from url calls
split = 4 * 1024
dest_blob.stage_block_from_url(block_id=1,
source_url=self.source_blob_url,
source_offset=0,
source_length=split)
dest_blob.stage_block_from_url(block_id=2,
source_url=self.source_blob_url,
source_offset=split,
source_length=split)
# Assert blocks
committed, uncommitted = dest_blob.get_block_list('all')
self.assertEqual(len(uncommitted), 2)
self.assertEqual(len(committed), 0)
# Act part 2: commit the blocks
dest_blob.commit_block_list(['1', '2'])
# Assert destination blob has right content
content = dest_blob.download_blob().readall()
self.assertEqual(len(content), 8 * 1024)
self.assertEqual(content, self.source_blob_data)
@GlobalStorageAccountPreparer()
def test_put_block_from_url_and_validate_content_md5(
self, resource_group, location, storage_account,
storage_account_key):
self._setup(storage_account, storage_account_key)
dest_blob_name = self.get_resource_name('destblob')
dest_blob = self.bsc.get_blob_client(self.container_name,
dest_blob_name)
src_md5 = StorageContentValidation.get_content_md5(
self.source_blob_data)
# Act part 1: put block from url with md5 validation
dest_blob.stage_block_from_url(block_id=1,
source_url=self.source_blob_url,
source_content_md5=src_md5,
source_offset=0,
source_length=8 * 1024)
# Assert block was staged
committed, uncommitted = dest_blob.get_block_list('all')
self.assertEqual(len(uncommitted), 1)
self.assertEqual(len(committed), 0)
# Act part 2: put block from url with wrong md5
fake_md5 = StorageContentValidation.get_content_md5(b"POTATO")
with self.assertRaises(HttpResponseError) as error:
dest_blob.stage_block_from_url(block_id=2,
source_url=self.source_blob_url,
source_content_md5=fake_md5,
source_offset=0,
source_length=8 * 1024)
self.assertEqual(error.exception.error_code,
StorageErrorCode.md5_mismatch)
# Assert block was not staged
committed, uncommitted = dest_blob.get_block_list('all')
self.assertEqual(len(uncommitted), 1)
self.assertEqual(len(committed), 0)
@GlobalStorageAccountPreparer()
def test_copy_blob_sync(self, resource_group, location, storage_account,
storage_account_key):
self._setup(storage_account, storage_account_key)
dest_blob_name = self.get_resource_name('destblob')
dest_blob = self.bsc.get_blob_client(self.container_name,
dest_blob_name)
# Act
copy_props = dest_blob.start_copy_from_url(self.source_blob_url,
requires_sync=True)
# Assert
self.assertIsNotNone(copy_props)
self.assertIsNotNone(copy_props['copy_id'])
self.assertEqual('success', copy_props['copy_status'])
# Verify content
content = dest_blob.download_blob().readall()
self.assertEqual(self.source_blob_data, content)
@pytest.mark.playback_test_only
@GlobalStorageAccountPreparer()
def test_sync_copy_blob_returns_vid(self, resource_group, location,
storage_account, storage_account_key):
self._setup(storage_account, storage_account_key)
dest_blob_name = self.get_resource_name('destblob')
dest_blob = self.bsc.get_blob_client(self.container_name,
dest_blob_name)
# Act
copy_props = dest_blob.start_copy_from_url(self.source_blob_url,
requires_sync=True)
# Assert
self.assertIsNotNone(copy_props['version_id'])
self.assertIsNotNone(copy_props)
self.assertIsNotNone(copy_props['copy_id'])
self.assertEqual('success', copy_props['copy_status'])
# Verify content
content = dest_blob.download_blob().readall()
self.assertEqual(self.source_blob_data, content)
class StorageBlockBlobTest(StorageTestCase):
def setUp(self):
super(StorageBlockBlobTest, self).setUp()
url = self._get_account_url()
# test chunking functionality by reducing the size of each chunk,
# otherwise the tests would take too long to execute
self.bsc = BlobServiceClient(
url,
credential=self.settings.STORAGE_ACCOUNT_KEY,
connection_data_block_size=4 * 1024,
max_single_put_size=32 * 1024,
max_block_size=4 * 1024)
self.config = self.bsc._config
self.container_name = self.get_resource_name('utcontainer')
if not self.is_playback():
self.bsc.create_container(self.container_name)
def tearDown(self):
if not self.is_playback():
try:
self.bsc.delete_container(self.container_name)
except:
pass
if os.path.isfile(FILE_PATH):
try:
os.remove(FILE_PATH)
except:
pass
return super(StorageBlockBlobTest, self).tearDown()
#--Helpers-----------------------------------------------------------------
def _get_blob_reference(self):
return self.get_resource_name(TEST_BLOB_PREFIX)
def _create_blob(self):
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
blob.upload_blob(b'')
return blob
def assertBlobEqual(self, container_name, blob_name, expected_data):
blob = self.bsc.get_blob_client(container_name, blob_name)
actual_data = blob.download_blob()
self.assertEqual(b"".join(list(actual_data)), expected_data)
class NonSeekableFile(object):
def __init__(self, wrapped_file):
self.wrapped_file = wrapped_file
def write(self, data):
self.wrapped_file.write(data)
def read(self, count):
return self.wrapped_file.read(count)
#--Test cases for block blobs --------------------------------------------
@record
def test_put_block(self):
# Arrange
blob = self._create_blob()
# Act
for i in range(5):
resp = blob.stage_block(i, 'block {0}'.format(i).encode('utf-8'))
self.assertIsNone(resp)
# Assert
@record
def test_put_block_unicode(self):
# Arrange
blob = self._create_blob()
# Act
resp = blob.stage_block('1', u'啊齄丂狛狜')
self.assertIsNone(resp)
# Assert
@record
def test_put_block_with_md5(self):
# Arrange
blob = self._create_blob()
# Act
blob.stage_block(1, b'block', validate_content=True)
# Assert
@record
def test_put_block_list(self):
# Arrange
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
blob.stage_block('1', b'AAA')
blob.stage_block('2', b'BBB')
blob.stage_block('3', b'CCC')
# Act
block_list = [
BlobBlock(block_id='1'),
BlobBlock(block_id='2'),
BlobBlock(block_id='3')
]
put_block_list_resp = blob.commit_block_list(block_list)
# Assert
content = blob.download_blob()
self.assertEqual(b"".join(list(content)), b'AAABBBCCC')
self.assertEqual(content.properties.etag,
put_block_list_resp.get('etag'))
self.assertEqual(content.properties.last_modified,
put_block_list_resp.get('last_modified'))
@record
def test_put_block_list_invalid_block_id(self):
# Arrange
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
blob.stage_block('1', b'AAA')
blob.stage_block('2', b'BBB')
blob.stage_block('3', b'CCC')
# Act
try:
block_list = [
BlobBlock(block_id='1'),
BlobBlock(block_id='2'),
BlobBlock(block_id='4')
]
blob.commit_block_list(block_list)
self.fail()
except HttpResponseError as e:
self.assertGreaterEqual(
str(e).find('specified block list is invalid'), 0)
# Assert
@record
def test_put_block_list_with_md5(self):
# Arrange
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
blob.stage_block('1', b'AAA')
blob.stage_block('2', b'BBB')
blob.stage_block('3', b'CCC')
# Act
block_list = [
BlobBlock(block_id='1'),
BlobBlock(block_id='2'),
BlobBlock(block_id='3')
]
blob.commit_block_list(block_list, validate_content=True)
# Assert
@record
def test_get_block_list_no_blocks(self):
# Arrange
blob = self._create_blob()
# Act
block_list = blob.get_block_list('all')
# Assert
self.assertIsNotNone(block_list)
self.assertEqual(len(block_list[1]), 0)
self.assertEqual(len(block_list[0]), 0)
@record
def test_get_block_list_uncommitted_blocks(self):
# Arrange
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
blob.stage_block('1', b'AAA')
blob.stage_block('2', b'BBB')
blob.stage_block('3', b'CCC')
# Act
block_list = blob.get_block_list('uncommitted')
# Assert
self.assertIsNotNone(block_list)
self.assertEqual(len(block_list), 2)
self.assertEqual(len(block_list[1]), 3)
self.assertEqual(len(block_list[0]), 0)
self.assertEqual(block_list[1][0].id, '1')
self.assertEqual(block_list[1][0].size, 3)
self.assertEqual(block_list[1][1].id, '2')
self.assertEqual(block_list[1][1].size, 3)
self.assertEqual(block_list[1][2].id, '3')
self.assertEqual(block_list[1][2].size, 3)
@record
def test_get_block_list_committed_blocks(self):
# Arrange
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
blob.stage_block('1', b'AAA')
blob.stage_block('2', b'BBB')
blob.stage_block('3', b'CCC')
block_list = [
BlobBlock(block_id='1'),
BlobBlock(block_id='2'),
BlobBlock(block_id='3')
]
blob.commit_block_list(block_list)
# Act
block_list = blob.get_block_list('committed')
# Assert
self.assertIsNotNone(block_list)
self.assertEqual(len(block_list), 2)
self.assertEqual(len(block_list[1]), 0)
self.assertEqual(len(block_list[0]), 3)
self.assertEqual(block_list[0][0].id, '1')
self.assertEqual(block_list[0][0].size, 3)
self.assertEqual(block_list[0][1].id, '2')
self.assertEqual(block_list[0][1].size, 3)
self.assertEqual(block_list[0][2].id, '3')
self.assertEqual(block_list[0][2].size, 3)
@record
def test_create_small_block_blob_with_no_overwrite(self):
# Arrange
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data1 = b'hello world'
data2 = b'hello second world'
# Act
create_resp = blob.upload_blob(data1, overwrite=True)
with self.assertRaises(ResourceExistsError):
blob.upload_blob(data2, overwrite=False)
props = blob.get_blob_properties()
# Assert
self.assertBlobEqual(self.container_name, blob_name, data1)
self.assertEqual(props.etag, create_resp.get('etag'))
self.assertEqual(props.last_modified, create_resp.get('last_modified'))
self.assertEqual(props.blob_type, BlobType.BlockBlob)
@record
def test_create_small_block_blob_with_overwrite(self):
# Arrange
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data1 = b'hello world'
data2 = b'hello second world'
# Act
create_resp = blob.upload_blob(data1, overwrite=True)
update_resp = blob.upload_blob(data2, overwrite=True)
props = blob.get_blob_properties()
# Assert
self.assertBlobEqual(self.container_name, blob_name, data2)
self.assertEqual(props.etag, update_resp.get('etag'))
self.assertEqual(props.last_modified, update_resp.get('last_modified'))
self.assertEqual(props.blob_type, BlobType.BlockBlob)
@record
def test_create_large_block_blob_with_no_overwrite(self):
# Arrange
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data1 = self.get_random_bytes(LARGE_BLOB_SIZE)
data2 = self.get_random_bytes(LARGE_BLOB_SIZE)
# Act
create_resp = blob.upload_blob(data1,
overwrite=True,
metadata={'BlobData': 'Data1'})
with self.assertRaises(ResourceExistsError):
blob.upload_blob(data2,
overwrite=False,
metadata={'BlobData': 'Data2'})
props = blob.get_blob_properties()
# Assert
self.assertBlobEqual(self.container_name, blob_name, data1)
self.assertEqual(props.etag, create_resp.get('etag'))
self.assertEqual(props.last_modified, create_resp.get('last_modified'))
self.assertEqual(props.blob_type, BlobType.BlockBlob)
self.assertEqual(props.metadata, {'BlobData': 'Data1'})
self.assertEqual(props.size, LARGE_BLOB_SIZE)
@record
def test_create_large_block_blob_with_overwrite(self):
# Arrange
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data1 = self.get_random_bytes(LARGE_BLOB_SIZE)
data2 = self.get_random_bytes(LARGE_BLOB_SIZE + 512)
# Act
create_resp = blob.upload_blob(data1,
overwrite=True,
metadata={'BlobData': 'Data1'})
update_resp = blob.upload_blob(data2,
overwrite=True,
metadata={'BlobData': 'Data2'})
props = blob.get_blob_properties()
# Assert
self.assertBlobEqual(self.container_name, blob_name, data2)
self.assertEqual(props.etag, update_resp.get('etag'))
self.assertEqual(props.last_modified, update_resp.get('last_modified'))
self.assertEqual(props.blob_type, BlobType.BlockBlob)
self.assertEqual(props.metadata, {'BlobData': 'Data2'})
self.assertEqual(props.size, LARGE_BLOB_SIZE + 512)
@record
def test_create_blob_from_bytes_single_put(self):
# Arrange
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = b'hello world'
# Act
create_resp = blob.upload_blob(data)
props = blob.get_blob_properties()
# Assert
self.assertBlobEqual(self.container_name, blob_name, data)
self.assertEqual(props.etag, create_resp.get('etag'))
self.assertEqual(props.last_modified, create_resp.get('last_modified'))
@record
def test_create_blob_from_0_bytes(self):
# Arrange
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = b''
# Act
create_resp = blob.upload_blob(data)
props = blob.get_blob_properties()
# Assert
self.assertBlobEqual(self.container_name, blob_name, data)
self.assertEqual(props.etag, create_resp.get('etag'))
self.assertEqual(props.last_modified, create_resp.get('last_modified'))
@record
def test_create_from_bytes_blob_unicode(self):
# Arrange
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = u'hello world'
# Act
create_resp = blob.upload_blob(data)
props = blob.get_blob_properties()
# Assert
self.assertBlobEqual(self.container_name, blob_name, data)
self.assertEqual(props.etag, create_resp.get('etag'))
self.assertEqual(props.last_modified, create_resp.get('last_modified'))
@record
def test_create_from_bytes_blob_unicode(self):
# Arrange
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
# Act
data = u'hello world'
create_resp = blob.upload_blob(data)
props = blob.get_blob_properties()
# Assert
self.assertBlobEqual(self.container_name, blob_name,
data.encode('utf-8'))
self.assertEqual(props.etag, create_resp.get('etag'))
self.assertEqual(props.last_modified, create_resp.get('last_modified'))
def test_create_from_bytes_blob_with_lease_id(self):
# parallel tests introduce random order of requests, can only run live
if TestMode.need_recording_file(self.test_mode):
return
# Arrange
blob = self._create_blob()
data = self.get_random_bytes(LARGE_BLOB_SIZE)
lease = blob.acquire_lease()
# Act
create_resp = blob.upload_blob(data, lease=lease)
# Assert
output = blob.download_blob(lease=lease)
self.assertEqual(b"".join(list(output)), data)
self.assertEqual(output.properties.etag, create_resp.get('etag'))
self.assertEqual(output.properties.last_modified,
create_resp.get('last_modified'))
def test_create_blob_from_bytes_with_metadata(self):
# parallel tests introduce random order of requests, can only run live
if TestMode.need_recording_file(self.test_mode):
return
# Arrange
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = self.get_random_bytes(LARGE_BLOB_SIZE)
metadata = {'hello': 'world', 'number': '42'}
# Act
blob.upload_blob(data, metadata=metadata)
# Assert
md = blob.get_blob_properties().metadata
self.assertDictEqual(md, metadata)
def test_create_blob_from_bytes_with_properties(self):
# parallel tests introduce random order of requests, can only run live
if TestMode.need_recording_file(self.test_mode):
return
# Arrange
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = self.get_random_bytes(LARGE_BLOB_SIZE)
# Act
content_settings = ContentSettings(content_type='image/png',
content_language='spanish')
blob.upload_blob(data, content_settings=content_settings)
# Assert
self.assertBlobEqual(self.container_name, blob_name, data)
properties = blob.get_blob_properties()
self.assertEqual(properties.content_settings.content_type,
content_settings.content_type)
self.assertEqual(properties.content_settings.content_language,
content_settings.content_language)
def test_create_blob_from_bytes_with_progress(self):
# parallel tests introduce random order of requests, can only run live
if TestMode.need_recording_file(self.test_mode):
return
# Arrange
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = self.get_random_bytes(LARGE_BLOB_SIZE)
# Act
progress = []
def callback(response):
current = response.context['upload_stream_current']
total = response.context['data_stream_total']
if current is not None:
progress.append((current, total))
create_resp = blob.upload_blob(data, raw_response_hook=callback)
props = blob.get_blob_properties()
# Assert
self.assertBlobEqual(self.container_name, blob_name, data)
self.assert_upload_progress(len(data), self.config.max_block_size,
progress)
self.assertEqual(props.etag, create_resp.get('etag'))
self.assertEqual(props.last_modified, create_resp.get('last_modified'))
def test_create_blob_from_bytes_with_index(self):
# parallel tests introduce random order of requests, can only run live
if TestMode.need_recording_file(self.test_mode):
return
# Arrange
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = self.get_random_bytes(LARGE_BLOB_SIZE)
# Act
blob.upload_blob(data[3:])
# Assert
self.assertEqual(data[3:], b"".join(list(blob.download_blob())))
@record
def test_create_blob_from_bytes_with_index_and_count(self):
# Arrange
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = self.get_random_bytes(LARGE_BLOB_SIZE)
# Act
blob.upload_blob(data[3:], length=5)
# Assert
self.assertEqual(data[3:8], b"".join(list(blob.download_blob())))
@record
def test_create_blob_from_bytes_with_index_and_count_and_properties(self):
# Arrange
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = self.get_random_bytes(LARGE_BLOB_SIZE)
# Act
content_settings = ContentSettings(content_type='image/png',
content_language='spanish')
blob.upload_blob(data[3:], length=5, content_settings=content_settings)
# Assert
self.assertEqual(data[3:8], b"".join(list(blob.download_blob())))
properties = blob.get_blob_properties()
self.assertEqual(properties.content_settings.content_type,
content_settings.content_type)
self.assertEqual(properties.content_settings.content_language,
content_settings.content_language)
@record
def test_create_blob_from_bytes_non_parallel(self):
# Arrange
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = self.get_random_bytes(LARGE_BLOB_SIZE)
# Act
blob.upload_blob(data, length=LARGE_BLOB_SIZE, max_connections=1)
# Assert
self.assertBlobEqual(self.container_name, blob.blob_name, data)
def test_create_blob_from_path(self):
# parallel tests introduce random order of requests, can only run live
if TestMode.need_recording_file(self.test_mode):
return
# Arrange
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = self.get_random_bytes(LARGE_BLOB_SIZE)
with open(FILE_PATH, 'wb') as stream:
stream.write(data)
# Act
with open(FILE_PATH, 'rb') as stream:
create_resp = blob.upload_blob(stream)
props = blob.get_blob_properties()
# Assert
self.assertBlobEqual(self.container_name, blob_name, data)
self.assertEqual(props.etag, create_resp.get('etag'))
self.assertEqual(props.last_modified, create_resp.get('last_modified'))
@record
def test_create_blob_from_path_non_parallel(self):
# Arrange
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = self.get_random_bytes(100)
with open(FILE_PATH, 'wb') as stream:
stream.write(data)
# Act
with open(FILE_PATH, 'rb') as stream:
create_resp = blob.upload_blob(stream,
length=100,
max_connections=1)
props = blob.get_blob_properties()
# Assert
self.assertBlobEqual(self.container_name, blob_name, data)
self.assertEqual(props.etag, create_resp.get('etag'))
self.assertEqual(props.last_modified, create_resp.get('last_modified'))
def test_create_blob_from_path_with_progress(self):
# parallel tests introduce random order of requests, can only run live
if TestMode.need_recording_file(self.test_mode):
return
# Arrange
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = self.get_random_bytes(LARGE_BLOB_SIZE)
with open(FILE_PATH, 'wb') as stream:
stream.write(data)
# Act
progress = []
def callback(response):
current = response.context['upload_stream_current']
total = response.context['data_stream_total']
if current is not None:
progress.append((current, total))
with open(FILE_PATH, 'rb') as stream:
blob.upload_blob(stream, raw_response_hook=callback)
# Assert
self.assertBlobEqual(self.container_name, blob_name, data)
self.assert_upload_progress(len(data), self.config.max_block_size,
progress)
def test_create_blob_from_path_with_properties(self):
# parallel tests introduce random order of requests, can only run live
if TestMode.need_recording_file(self.test_mode):
return
# Arrange
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = self.get_random_bytes(LARGE_BLOB_SIZE)
with open(FILE_PATH, 'wb') as stream:
stream.write(data)
# Act
content_settings = ContentSettings(content_type='image/png',
content_language='spanish')
with open(FILE_PATH, 'rb') as stream:
blob.upload_blob(stream, content_settings=content_settings)
# Assert
self.assertBlobEqual(self.container_name, blob_name, data)
properties = blob.get_blob_properties()
self.assertEqual(properties.content_settings.content_type,
content_settings.content_type)
self.assertEqual(properties.content_settings.content_language,
content_settings.content_language)
def test_create_blob_from_stream_chunked_upload(self):
# parallel tests introduce random order of requests, can only run live
if TestMode.need_recording_file(self.test_mode):
return
# Arrange
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = self.get_random_bytes(LARGE_BLOB_SIZE)
with open(FILE_PATH, 'wb') as stream:
stream.write(data)
# Act
with open(FILE_PATH, 'rb') as stream:
create_resp = blob.upload_blob(stream)
props = blob.get_blob_properties()
# Assert
self.assertBlobEqual(self.container_name, blob_name, data)
self.assertEqual(props.etag, create_resp.get('etag'))
self.assertEqual(props.last_modified, create_resp.get('last_modified'))
def test_create_blob_from_stream_non_seekable_chunked_upload_known_size(
self):
# parallel tests introduce random order of requests, can only run live
if TestMode.need_recording_file(self.test_mode):
return
# Arrange
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = self.get_random_bytes(LARGE_BLOB_SIZE)
blob_size = len(data) - 66
with open(FILE_PATH, 'wb') as stream:
stream.write(data)
# Act
with open(FILE_PATH, 'rb') as stream:
non_seekable_file = StorageBlockBlobTest.NonSeekableFile(stream)
blob.upload_blob(non_seekable_file,
length=blob_size,
max_connections=1)
# Assert
self.assertBlobEqual(self.container_name, blob_name, data[:blob_size])
def test_create_blob_from_stream_non_seekable_chunked_upload_unknown_size(
self):
# parallel tests introduce random order of requests, can only run live
if TestMode.need_recording_file(self.test_mode):
return
# Arrange
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = self.get_random_bytes(LARGE_BLOB_SIZE)
with open(FILE_PATH, 'wb') as stream:
stream.write(data)
# Act
with open(FILE_PATH, 'rb') as stream:
non_seekable_file = StorageBlockBlobTest.NonSeekableFile(stream)
blob.upload_blob(non_seekable_file, max_connections=1)
# Assert
self.assertBlobEqual(self.container_name, blob_name, data)
def test_create_blob_from_stream_with_progress_chunked_upload(self):
# parallel tests introduce random order of requests, can only run live
if TestMode.need_recording_file(self.test_mode):
return
# Arrange
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = self.get_random_bytes(LARGE_BLOB_SIZE)
with open(FILE_PATH, 'wb') as stream:
stream.write(data)
# Act
progress = []
def callback(response):
current = response.context['upload_stream_current']
total = response.context['data_stream_total']
if current is not None:
progress.append((current, total))
with open(FILE_PATH, 'rb') as stream:
blob.upload_blob(stream, raw_response_hook=callback)
# Assert
self.assertBlobEqual(self.container_name, blob_name, data)
self.assert_upload_progress(len(data), self.config.max_block_size,
progress)
def test_create_blob_from_stream_chunked_upload_with_count(self):
# parallel tests introduce random order of requests, can only run live
if TestMode.need_recording_file(self.test_mode):
return
# Arrange
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = self.get_random_bytes(LARGE_BLOB_SIZE)
with open(FILE_PATH, 'wb') as stream:
stream.write(data)
# Act
blob_size = len(data) - 301
with open(FILE_PATH, 'rb') as stream:
resp = blob.upload_blob(stream, length=blob_size)
# Assert
self.assertBlobEqual(self.container_name, blob_name, data[:blob_size])
def test_create_blob_from_stream_chunked_upload_with_count_and_properties(
self):
# parallel tests introduce random order of requests, can only run live
if TestMode.need_recording_file(self.test_mode):
return
# Arrange
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = self.get_random_bytes(LARGE_BLOB_SIZE)
with open(FILE_PATH, 'wb') as stream:
stream.write(data)
# Act
content_settings = ContentSettings(content_type='image/png',
content_language='spanish')
blob_size = len(data) - 301
with open(FILE_PATH, 'rb') as stream:
blob.upload_blob(stream,
length=blob_size,
content_settings=content_settings)
# Assert
self.assertBlobEqual(self.container_name, blob_name, data[:blob_size])
properties = blob.get_blob_properties()
self.assertEqual(properties.content_settings.content_type,
content_settings.content_type)
self.assertEqual(properties.content_settings.content_language,
content_settings.content_language)
def test_create_blob_from_stream_chunked_upload_with_properties(self):
# parallel tests introduce random order of requests, can only run live
if TestMode.need_recording_file(self.test_mode):
return
# Arrange
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = self.get_random_bytes(LARGE_BLOB_SIZE)
with open(FILE_PATH, 'wb') as stream:
stream.write(data)
# Act
content_settings = ContentSettings(content_type='image/png',
content_language='spanish')
with open(FILE_PATH, 'rb') as stream:
blob.upload_blob(stream, content_settings=content_settings)
# Assert
self.assertBlobEqual(self.container_name, blob_name, data)
properties = blob.get_blob_properties()
self.assertEqual(properties.content_settings.content_type,
content_settings.content_type)
self.assertEqual(properties.content_settings.content_language,
content_settings.content_language)
@record
def test_create_blob_from_text(self):
# Arrange
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
text = u'hello 啊齄丂狛狜 world'
data = text.encode('utf-8')
# Act
create_resp = blob.upload_blob(text)
props = blob.get_blob_properties()
# Assert
self.assertBlobEqual(self.container_name, blob_name, data)
self.assertEqual(props.etag, create_resp.get('etag'))
self.assertEqual(props.last_modified, create_resp.get('last_modified'))
@record
def test_create_blob_from_text_with_encoding(self):
# Arrange
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
text = u'hello 啊齄丂狛狜 world'
data = text.encode('utf-16')
# Act
blob.upload_blob(text, encoding='utf-16')
# Assert
self.assertBlobEqual(self.container_name, blob_name, data)
@record
def test_create_blob_from_text_with_encoding_and_progress(self):
# Arrange
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
text = u'hello 啊齄丂狛狜 world'
data = text.encode('utf-16')
# Act
progress = []
def callback(response):
current = response.context['upload_stream_current']
total = response.context['data_stream_total']
if current is not None:
progress.append((current, total))
blob.upload_blob(text, encoding='utf-16', raw_response_hook=callback)
# Assert
self.assertBlobEqual(self.container_name, blob_name, data)
self.assert_upload_progress(len(data), self.config.max_block_size,
progress)
def test_create_blob_from_text_chunked_upload(self):
# parallel tests introduce random order of requests, can only run live
if TestMode.need_recording_file(self.test_mode):
return
# Arrange
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = self.get_random_text_data(LARGE_BLOB_SIZE)
encoded_data = data.encode('utf-8')
# Act
blob.upload_blob(data)
# Assert
self.assertBlobEqual(self.container_name, blob_name, encoded_data)
# Assert
self.assertBlobEqual(self.container_name, blob_name, encoded_data)
@record
def test_create_blob_with_md5(self):
# Arrange
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = b'hello world'
# Act
blob.upload_blob(data, validate_content=True)
# Assert
def test_create_blob_with_md5_chunked(self):
# parallel tests introduce random order of requests, can only run live
if TestMode.need_recording_file(self.test_mode):
return
# Arrange
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = self.get_random_bytes(LARGE_BLOB_SIZE)
# Act
blob.upload_blob(data, validate_content=True)
class StorageBlockBlobTest(StorageTestCase):
def _setup(self, storage_account, key):
# test chunking functionality by reducing the size of each chunk,
# otherwise the tests would take too long to execute
self.bsc = BlobServiceClient(
self.account_url(storage_account, "blob"),
credential=key,
connection_data_block_size=4 * 1024,
max_single_put_size=32 * 1024,
max_block_size=4 * 1024)
self.config = self.bsc._config
self.container_name = self.get_resource_name('utcontainer')
if self.is_live:
self.bsc.create_container(self.container_name)
def _teardown(self, FILE_PATH):
if os.path.isfile(FILE_PATH):
try:
os.remove(FILE_PATH)
except:
pass
#--Helpers-----------------------------------------------------------------
def _get_blob_reference(self):
return self.get_resource_name(TEST_BLOB_PREFIX)
def _create_blob(self):
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
blob.upload_blob(b'')
return blob
def assertBlobEqual(self, container_name, blob_name, expected_data):
blob = self.bsc.get_blob_client(container_name, blob_name)
actual_data = blob.download_blob()
self.assertEqual(actual_data.readall(), expected_data)
class NonSeekableFile(object):
def __init__(self, wrapped_file):
self.wrapped_file = wrapped_file
def write(self, data):
self.wrapped_file.write(data)
def read(self, count):
return self.wrapped_file.read(count)
#--Test cases for block blobs --------------------------------------------
@GlobalStorageAccountPreparer()
def test_put_block(self, resource_group, location, storage_account, storage_account_key):
self._setup(storage_account, storage_account_key)
blob = self._create_blob()
# Act
for i in range(5):
headers = blob.stage_block(i, 'block {0}'.format(i).encode('utf-8'))
self.assertIn('content_crc64', headers)
# Assert
@GlobalStorageAccountPreparer()
def test_put_block_with_response(self, resource_group, location, storage_account, storage_account_key):
self._setup(storage_account.name, storage_account_key)
blob = self._create_blob()
def return_response(resp, _, headers):
return (resp, headers)
# Act
resp, headers = blob.stage_block(0, 'block 0', cls=return_response)
# Assert
self.assertEqual(201, resp.status_code)
self.assertIn('x-ms-content-crc64', headers)
@GlobalStorageAccountPreparer()
def test_put_block_unicode(self, resource_group, location, storage_account, storage_account_key):
self._setup(storage_account, storage_account_key)
blob = self._create_blob()
# Act
headers = blob.stage_block('1', u'啊齄丂狛狜')
self.assertIn('content_crc64', headers)
# Assert
@GlobalStorageAccountPreparer()
def test_put_block_with_md5(self, resource_group, location, storage_account, storage_account_key):
self._setup(storage_account, storage_account_key)
blob = self._create_blob()
# Act
blob.stage_block(1, b'block', validate_content=True)
# Assert
@GlobalStorageAccountPreparer()
def test_put_block_list(self, resource_group, location, storage_account, storage_account_key):
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
blob.stage_block('1', b'AAA')
blob.stage_block('2', b'BBB')
blob.stage_block('3', b'CCC')
# Act
block_list = [BlobBlock(block_id='1'), BlobBlock(block_id='2'), BlobBlock(block_id='3')]
put_block_list_resp = blob.commit_block_list(block_list)
# Assert
content = blob.download_blob()
self.assertEqual(content.readall(), b'AAABBBCCC')
self.assertEqual(content.properties.etag, put_block_list_resp.get('etag'))
self.assertEqual(content.properties.last_modified, put_block_list_resp.get('last_modified'))
@GlobalStorageAccountPreparer()
def test_put_block_list_invalid_block_id(self, resource_group, location, storage_account, storage_account_key):
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
blob.stage_block('1', b'AAA')
blob.stage_block('2', b'BBB')
blob.stage_block('3', b'CCC')
# Act
try:
block_list = [BlobBlock(block_id='1'), BlobBlock(block_id='2'), BlobBlock(block_id='4')]
blob.commit_block_list(block_list)
self.fail()
except HttpResponseError as e:
self.assertGreaterEqual(str(e).find('specified block list is invalid'), 0)
# Assert
@GlobalStorageAccountPreparer()
def test_put_block_list_with_md5(self, resource_group, location, storage_account, storage_account_key):
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
blob.stage_block('1', b'AAA')
blob.stage_block('2', b'BBB')
blob.stage_block('3', b'CCC')
# Act
block_list = [BlobBlock(block_id='1'), BlobBlock(block_id='2'), BlobBlock(block_id='3')]
blob.commit_block_list(block_list, validate_content=True)
# Assert
@GlobalStorageAccountPreparer()
def test_put_block_list_with_blob_tier_specified(self, resource_group, location, storage_account, storage_account_key):
# Arrange
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob_client = self.bsc.get_blob_client(self.container_name, blob_name)
blob_client.stage_block('1', b'AAA')
blob_client.stage_block('2', b'BBB')
blob_client.stage_block('3', b'CCC')
blob_tier = StandardBlobTier.Cool
# Act
block_list = [BlobBlock(block_id='1'), BlobBlock(block_id='2'), BlobBlock(block_id='3')]
blob_client.commit_block_list(block_list,
standard_blob_tier=blob_tier)
# Assert
blob_properties = blob_client.get_blob_properties()
self.assertEqual(blob_properties.blob_tier, blob_tier)
@GlobalStorageAccountPreparer()
def test_get_block_list_no_blocks(self, resource_group, location, storage_account, storage_account_key):
self._setup(storage_account, storage_account_key)
blob = self._create_blob()
# Act
block_list = blob.get_block_list('all')
# Assert
self.assertIsNotNone(block_list)
self.assertEqual(len(block_list[1]), 0)
self.assertEqual(len(block_list[0]), 0)
@GlobalStorageAccountPreparer()
def test_get_block_list_uncommitted_blocks(self, resource_group, location, storage_account, storage_account_key):
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
blob.stage_block('1', b'AAA')
blob.stage_block('2', b'BBB')
blob.stage_block('3', b'CCC')
# Act
block_list = blob.get_block_list('uncommitted')
# Assert
self.assertIsNotNone(block_list)
self.assertEqual(len(block_list), 2)
self.assertEqual(len(block_list[1]), 3)
self.assertEqual(len(block_list[0]), 0)
self.assertEqual(block_list[1][0].id, '1')
self.assertEqual(block_list[1][0].size, 3)
self.assertEqual(block_list[1][1].id, '2')
self.assertEqual(block_list[1][1].size, 3)
self.assertEqual(block_list[1][2].id, '3')
self.assertEqual(block_list[1][2].size, 3)
@GlobalStorageAccountPreparer()
def test_get_block_list_committed_blocks(self, resource_group, location, storage_account, storage_account_key):
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
blob.stage_block('1', b'AAA')
blob.stage_block('2', b'BBB')
blob.stage_block('3', b'CCC')
block_list = [BlobBlock(block_id='1'), BlobBlock(block_id='2'), BlobBlock(block_id='3')]
blob.commit_block_list(block_list)
# Act
block_list = blob.get_block_list('committed')
# Assert
self.assertIsNotNone(block_list)
self.assertEqual(len(block_list), 2)
self.assertEqual(len(block_list[1]), 0)
self.assertEqual(len(block_list[0]), 3)
self.assertEqual(block_list[0][0].id, '1')
self.assertEqual(block_list[0][0].size, 3)
self.assertEqual(block_list[0][1].id, '2')
self.assertEqual(block_list[0][1].size, 3)
self.assertEqual(block_list[0][2].id, '3')
self.assertEqual(block_list[0][2].size, 3)
@GlobalStorageAccountPreparer()
def test_create_small_block_blob_with_no_overwrite(self, resource_group, location, storage_account, storage_account_key):
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data1 = b'hello world'
data2 = b'hello second world'
# Act
create_resp = blob.upload_blob(data1, overwrite=True)
with self.assertRaises(ResourceExistsError):
blob.upload_blob(data2, overwrite=False)
props = blob.get_blob_properties()
# Assert
self.assertBlobEqual(self.container_name, blob_name, data1)
self.assertEqual(props.etag, create_resp.get('etag'))
self.assertEqual(props.last_modified, create_resp.get('last_modified'))
self.assertEqual(props.blob_type, BlobType.BlockBlob)
@GlobalStorageAccountPreparer()
def test_create_small_block_blob_with_overwrite(self, resource_group, location, storage_account, storage_account_key):
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data1 = b'hello world'
data2 = b'hello second world'
# Act
create_resp = blob.upload_blob(data1, overwrite=True)
update_resp = blob.upload_blob(data2, overwrite=True)
props = blob.get_blob_properties()
# Assert
self.assertBlobEqual(self.container_name, blob_name, data2)
self.assertEqual(props.etag, update_resp.get('etag'))
self.assertEqual(props.last_modified, update_resp.get('last_modified'))
self.assertEqual(props.blob_type, BlobType.BlockBlob)
@GlobalStorageAccountPreparer()
def test_create_large_block_blob_with_no_overwrite(self, resource_group, location, storage_account, storage_account_key):
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data1 = self.get_random_bytes(LARGE_BLOB_SIZE)
data2 = self.get_random_bytes(LARGE_BLOB_SIZE)
# Act
create_resp = blob.upload_blob(data1, overwrite=True, metadata={'blobdata': 'data1'})
with self.assertRaises(ResourceExistsError):
blob.upload_blob(data2, overwrite=False, metadata={'blobdata': 'data2'})
props = blob.get_blob_properties()
# Assert
self.assertBlobEqual(self.container_name, blob_name, data1)
self.assertEqual(props.etag, create_resp.get('etag'))
self.assertEqual(props.last_modified, create_resp.get('last_modified'))
self.assertEqual(props.blob_type, BlobType.BlockBlob)
self.assertEqual(props.metadata, {'blobdata': 'data1'})
self.assertEqual(props.size, LARGE_BLOB_SIZE)
@GlobalStorageAccountPreparer()
def test_create_large_block_blob_with_overwrite(self, resource_group, location, storage_account, storage_account_key):
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data1 = self.get_random_bytes(LARGE_BLOB_SIZE)
data2 = self.get_random_bytes(LARGE_BLOB_SIZE + 512)
# Act
create_resp = blob.upload_blob(data1, overwrite=True, metadata={'blobdata': 'data1'})
update_resp = blob.upload_blob(data2, overwrite=True, metadata={'blobdata': 'data2'})
props = blob.get_blob_properties()
# Assert
self.assertBlobEqual(self.container_name, blob_name, data2)
self.assertEqual(props.etag, update_resp.get('etag'))
self.assertEqual(props.last_modified, update_resp.get('last_modified'))
self.assertEqual(props.blob_type, BlobType.BlockBlob)
self.assertEqual(props.metadata, {'blobdata': 'data2'})
self.assertEqual(props.size, LARGE_BLOB_SIZE + 512)
@GlobalStorageAccountPreparer()
def test_create_blob_from_bytes_single_put(self, resource_group, location, storage_account, storage_account_key):
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = b'hello world'
# Act
create_resp = blob.upload_blob(data)
props = blob.get_blob_properties()
# Assert
self.assertBlobEqual(self.container_name, blob_name, data)
self.assertEqual(props.etag, create_resp.get('etag'))
self.assertEqual(props.last_modified, create_resp.get('last_modified'))
@GlobalStorageAccountPreparer()
def test_create_blob_from_0_bytes(self, resource_group, location, storage_account, storage_account_key):
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = b''
# Act
create_resp = blob.upload_blob(data)
props = blob.get_blob_properties()
# Assert
self.assertBlobEqual(self.container_name, blob_name, data)
self.assertEqual(props.etag, create_resp.get('etag'))
self.assertEqual(props.last_modified, create_resp.get('last_modified'))
@GlobalStorageAccountPreparer()
def test_create_from_bytes_blob_unicode(self, resource_group, location, storage_account, storage_account_key):
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = u'hello world'
# Act
create_resp = blob.upload_blob(data)
props = blob.get_blob_properties()
# Assert
self.assertBlobEqual(self.container_name, blob_name, data)
self.assertEqual(props.etag, create_resp.get('etag'))
self.assertEqual(props.last_modified, create_resp.get('last_modified'))
@GlobalStorageAccountPreparer()
def test_create_from_bytes_blob_unicode(self, resource_group, location, storage_account, storage_account_key):
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
# Act
data = u'hello world'
create_resp = blob.upload_blob(data)
props = blob.get_blob_properties()
# Assert
self.assertBlobEqual(self.container_name, blob_name, data.encode('utf-8'))
self.assertEqual(props.etag, create_resp.get('etag'))
self.assertEqual(props.last_modified, create_resp.get('last_modified'))
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
def test_create_from_bytes_blob_with_lease_id(self, resource_group, location, storage_account, storage_account_key):
# parallel tests introduce random order of requests, can only run live
self._setup(storage_account, storage_account_key)
blob = self._create_blob()
data = self.get_random_bytes(LARGE_BLOB_SIZE)
lease = blob.acquire_lease()
# Act
create_resp = blob.upload_blob(data, lease=lease)
# Assert
output = blob.download_blob(lease=lease)
self.assertEqual(output.readall(), data)
self.assertEqual(output.properties.etag, create_resp.get('etag'))
self.assertEqual(output.properties.last_modified, create_resp.get('last_modified'))
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
def test_create_blob_from_bytes_with_metadata(self, resource_group, location, storage_account, storage_account_key):
# parallel tests introduce random order of requests, can only run live
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = self.get_random_bytes(LARGE_BLOB_SIZE)
metadata = {'hello': 'world', 'number': '42'}
# Act
blob.upload_blob(data, metadata=metadata)
# Assert
md = blob.get_blob_properties().metadata
self.assertDictEqual(md, metadata)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
def test_create_blob_from_bytes_with_properties(self, resource_group, location, storage_account, storage_account_key):
# parallel tests introduce random order of requests, can only run live
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = self.get_random_bytes(LARGE_BLOB_SIZE)
# Act
content_settings=ContentSettings(
content_type='image/png',
content_language='spanish')
blob.upload_blob(data, content_settings=content_settings)
# Assert
self.assertBlobEqual(self.container_name, blob_name, data)
properties = blob.get_blob_properties()
self.assertEqual(properties.content_settings.content_type, content_settings.content_type)
self.assertEqual(properties.content_settings.content_language, content_settings.content_language)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
def test_create_blob_from_bytes_with_progress(self, resource_group, location, storage_account, storage_account_key):
# parallel tests introduce random order of requests, can only run live
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = self.get_random_bytes(LARGE_BLOB_SIZE)
# Act
progress = []
def callback(response):
current = response.context['upload_stream_current']
total = response.context['data_stream_total']
if current is not None:
progress.append((current, total))
create_resp = blob.upload_blob(data, raw_response_hook=callback)
props = blob.get_blob_properties()
# Assert
self.assertBlobEqual(self.container_name, blob_name, data)
self.assert_upload_progress(len(data), self.config.max_block_size, progress)
self.assertEqual(props.etag, create_resp.get('etag'))
self.assertEqual(props.last_modified, create_resp.get('last_modified'))
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
def test_create_blob_from_bytes_with_index(self, resource_group, location, storage_account, storage_account_key):
# parallel tests introduce random order of requests, can only run live
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = self.get_random_bytes(LARGE_BLOB_SIZE)
# Act
blob.upload_blob(data[3:])
# Assert
self.assertEqual(data[3:], blob.download_blob().readall())
@GlobalStorageAccountPreparer()
def test_create_blob_from_bytes_with_index_and_count(self, resource_group, location, storage_account, storage_account_key):
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = self.get_random_bytes(LARGE_BLOB_SIZE)
# Act
blob.upload_blob(data[3:], length=5)
# Assert
self.assertEqual(data[3:8], blob.download_blob().readall())
@GlobalStorageAccountPreparer()
def test_create_blob_from_bytes_with_index_and_count_and_properties(self, resource_group, location, storage_account, storage_account_key):
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = self.get_random_bytes(LARGE_BLOB_SIZE)
# Act
content_settings=ContentSettings(
content_type='image/png',
content_language='spanish')
blob.upload_blob(data[3:], length=5, content_settings=content_settings)
# Assert
self.assertEqual(data[3:8], blob.download_blob().readall())
properties = blob.get_blob_properties()
self.assertEqual(properties.content_settings.content_type, content_settings.content_type)
self.assertEqual(properties.content_settings.content_language, content_settings.content_language)
@GlobalStorageAccountPreparer()
def test_create_blob_from_bytes_non_parallel(self, resource_group, location, storage_account, storage_account_key):
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = self.get_random_bytes(LARGE_BLOB_SIZE)
# Act
blob.upload_blob(data, length=LARGE_BLOB_SIZE, max_concurrency=1)
# Assert
self.assertBlobEqual(self.container_name, blob.blob_name, data)
@GlobalStorageAccountPreparer()
def test_create_blob_from_bytes_with_blob_tier_specified(self, resource_group, location, storage_account, storage_account_key):
# Arrange
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob_client = self.bsc.get_blob_client(self.container_name, blob_name)
data = b'hello world'
blob_tier = StandardBlobTier.Cool
# Act
blob_client.upload_blob(data, standard_blob_tier=blob_tier)
blob_properties = blob_client.get_blob_properties()
# Assert
self.assertEqual(blob_properties.blob_tier, blob_tier)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
def test_create_blob_from_path(self, resource_group, location, storage_account, storage_account_key):
# parallel tests introduce random order of requests, can only run live
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = self.get_random_bytes(LARGE_BLOB_SIZE)
FILE_PATH = 'create_blob_from_input.temp.{}.dat'.format(str(uuid.uuid4()))
with open(FILE_PATH, 'wb') as stream:
stream.write(data)
# Act
with open(FILE_PATH, 'rb') as stream:
create_resp = blob.upload_blob(stream)
props = blob.get_blob_properties()
# Assert
self.assertBlobEqual(self.container_name, blob_name, data)
self.assertEqual(props.etag, create_resp.get('etag'))
self.assertEqual(props.last_modified, create_resp.get('last_modified'))
self._teardown(FILE_PATH)
@GlobalStorageAccountPreparer()
def test_create_blob_from_path_non_parallel(self, resource_group, location, storage_account, storage_account_key):
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = self.get_random_bytes(100)
FILE_PATH = 'create_blob_from_path_non_par.temp.{}.dat'.format(str(uuid.uuid4()))
with open(FILE_PATH, 'wb') as stream:
stream.write(data)
# Act
with open(FILE_PATH, 'rb') as stream:
create_resp = blob.upload_blob(stream, length=100, max_concurrency=1)
props = blob.get_blob_properties()
# Assert
self.assertBlobEqual(self.container_name, blob_name, data)
self.assertEqual(props.etag, create_resp.get('etag'))
self.assertEqual(props.last_modified, create_resp.get('last_modified'))
self._teardown(FILE_PATH)
@GlobalStorageAccountPreparer()
def test_upload_blob_from_path_non_parallel_with_standard_blob_tier(self, resource_group, location, storage_account, storage_account_key):
# Arrange
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = self.get_random_bytes(100)
FILE_PATH = '_path_non_parallel_with_standard_blob.temp.{}.dat'.format(str(uuid.uuid4()))
with open(FILE_PATH, 'wb') as stream:
stream.write(data)
blob_tier = StandardBlobTier.Cool
# Act
with open(FILE_PATH, 'rb') as stream:
blob.upload_blob(stream, length=100, max_concurrency=1, standard_blob_tier=blob_tier)
props = blob.get_blob_properties()
# Assert
self.assertEqual(props.blob_tier, blob_tier)
self._teardown(FILE_PATH)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
def test_create_blob_from_path_with_progress(self, resource_group, location, storage_account, storage_account_key):
# parallel tests introduce random order of requests, can only run live
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = self.get_random_bytes(LARGE_BLOB_SIZE)
FILE_PATH = 'create_blob_from_path_with_progr.temp.{}.dat'.format(str(uuid.uuid4()))
with open(FILE_PATH, 'wb') as stream:
stream.write(data)
# Act
progress = []
def callback(response):
current = response.context['upload_stream_current']
total = response.context['data_stream_total']
if current is not None:
progress.append((current, total))
with open(FILE_PATH, 'rb') as stream:
blob.upload_blob(stream, raw_response_hook=callback)
# Assert
self.assertBlobEqual(self.container_name, blob_name, data)
self.assert_upload_progress(len(data), self.config.max_block_size, progress)
self._teardown(FILE_PATH)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
def test_create_blob_from_path_with_properties(self, resource_group, location, storage_account, storage_account_key):
# parallel tests introduce random order of requests, can only run live
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = self.get_random_bytes(LARGE_BLOB_SIZE)
FILE_PATH = 'blob_from_path_with_properties.temp.{}.dat'.format(str(uuid.uuid4()))
with open(FILE_PATH, 'wb') as stream:
stream.write(data)
# Act
content_settings=ContentSettings(
content_type='image/png',
content_language='spanish')
with open(FILE_PATH, 'rb') as stream:
blob.upload_blob(stream, content_settings=content_settings)
# Assert
self.assertBlobEqual(self.container_name, blob_name, data)
properties = blob.get_blob_properties()
self.assertEqual(properties.content_settings.content_type, content_settings.content_type)
self.assertEqual(properties.content_settings.content_language, content_settings.content_language)
self._teardown(FILE_PATH)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
def test_create_blob_from_stream_chunked_upload(self, resource_group, location, storage_account, storage_account_key):
# parallel tests introduce random order of requests, can only run live
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = self.get_random_bytes(LARGE_BLOB_SIZE)
FILE_PATH = 'blob_from_stream_chunked_up.temp.{}.dat'.format(str(uuid.uuid4()))
with open(FILE_PATH, 'wb') as stream:
stream.write(data)
# Act
with open(FILE_PATH, 'rb') as stream:
create_resp = blob.upload_blob(stream)
props = blob.get_blob_properties()
# Assert
self.assertBlobEqual(self.container_name, blob_name, data)
self.assertEqual(props.etag, create_resp.get('etag'))
self.assertEqual(props.last_modified, create_resp.get('last_modified'))
self._teardown(FILE_PATH)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
def test_create_frm_stream_nonseek_chunk_upld_knwn_size(self, resource_group, location, storage_account, storage_account_key):
# parallel tests introduce random order of requests, can only run live
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = self.get_random_bytes(LARGE_BLOB_SIZE)
blob_size = len(data) - 66
FILE_PATH = 'stream_nonseek_chunk_upld_knwn_size.temp.{}.dat'.format(str(uuid.uuid4()))
with open(FILE_PATH, 'wb') as stream:
stream.write(data)
# Act
with open(FILE_PATH, 'rb') as stream:
non_seekable_file = StorageBlockBlobTest.NonSeekableFile(stream)
blob.upload_blob(non_seekable_file, length=blob_size, max_concurrency=1)
# Assert
self.assertBlobEqual(self.container_name, blob_name, data[:blob_size])
self._teardown(FILE_PATH)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
def test_create_from_stream_nonseek_chunk_upld_unkwn_size(self, resource_group, location, storage_account, storage_account_key):
# parallel tests introduce random order of requests, can only run live
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = self.get_random_bytes(LARGE_BLOB_SIZE)
FILE_PATH = 'stream_nonseek_chunk_upld.temp.{}.dat'.format(str(uuid.uuid4()))
with open(FILE_PATH, 'wb') as stream:
stream.write(data)
# Act
with open(FILE_PATH, 'rb') as stream:
non_seekable_file = StorageBlockBlobTest.NonSeekableFile(stream)
blob.upload_blob(non_seekable_file, max_concurrency=1)
# Assert
self.assertBlobEqual(self.container_name, blob_name, data)
self._teardown(FILE_PATH)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
def test_create_blob_from_stream_with_progress_chunked_upload(self, resource_group, location, storage_account, storage_account_key):
# parallel tests introduce random order of requests, can only run live
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = self.get_random_bytes(LARGE_BLOB_SIZE)
FILE_PATH = 'stream_with_progress_chunked.temp.{}.dat'.format(str(uuid.uuid4()))
with open(FILE_PATH, 'wb') as stream:
stream.write(data)
# Act
progress = []
def callback(response):
current = response.context['upload_stream_current']
total = response.context['data_stream_total']
if current is not None:
progress.append((current, total))
with open(FILE_PATH, 'rb') as stream:
blob.upload_blob(stream, raw_response_hook=callback)
# Assert
self.assertBlobEqual(self.container_name, blob_name, data)
self.assert_upload_progress(len(data), self.config.max_block_size, progress)
self._teardown(FILE_PATH)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
def test_create_blob_from_stream_chunked_upload_with_count(self, resource_group, location, storage_account, storage_account_key):
# parallel tests introduce random order of requests, can only run live
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = self.get_random_bytes(LARGE_BLOB_SIZE)
FILE_PATH = 'chunked_upload_with_count.temp.{}.dat'.format(str(uuid.uuid4()))
with open(FILE_PATH, 'wb') as stream:
stream.write(data)
# Act
blob_size = len(data) - 301
with open(FILE_PATH, 'rb') as stream:
resp = blob.upload_blob(stream, length=blob_size)
# Assert
self.assertBlobEqual(self.container_name, blob_name, data[:blob_size])
self._teardown(FILE_PATH)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
def test_create_from_stream_chunk_upload_with_cntandrops(self, resource_group, location, storage_account, storage_account_key):
# parallel tests introduce random order of requests, can only run live
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = self.get_random_bytes(LARGE_BLOB_SIZE)
FILE_PATH = 'from_stream_chunk_upload_with_cntandrops.temp.{}.dat'.format(str(uuid.uuid4()))
with open(FILE_PATH, 'wb') as stream:
stream.write(data)
# Act
content_settings=ContentSettings(
content_type='image/png',
content_language='spanish')
blob_size = len(data) - 301
with open(FILE_PATH, 'rb') as stream:
blob.upload_blob(stream, length=blob_size, content_settings=content_settings)
# Assert
self.assertBlobEqual(self.container_name, blob_name, data[:blob_size])
properties = blob.get_blob_properties()
self.assertEqual(properties.content_settings.content_type, content_settings.content_type)
self.assertEqual(properties.content_settings.content_language, content_settings.content_language)
self._teardown(FILE_PATH)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
def test_create_blob_from_stream_chnked_upload_with_properties(self, resource_group, location, storage_account, storage_account_key):
# parallel tests introduce random order of requests, can only run live
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = self.get_random_bytes(LARGE_BLOB_SIZE)
FILE_PATH = 'chnked_upload_with_properti.temp.{}.dat'.format(str(uuid.uuid4()))
with open(FILE_PATH, 'wb') as stream:
stream.write(data)
# Act
content_settings=ContentSettings(
content_type='image/png',
content_language='spanish')
with open(FILE_PATH, 'rb') as stream:
blob.upload_blob(stream, content_settings=content_settings)
# Assert
self.assertBlobEqual(self.container_name, blob_name, data)
properties = blob.get_blob_properties()
self.assertEqual(properties.content_settings.content_type, content_settings.content_type)
self.assertEqual(properties.content_settings.content_language, content_settings.content_language)
self._teardown(FILE_PATH)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
def test_create_blob_from_stream_chunked_upload_with_properties(self, resource_group, location, storage_account, storage_account_key):
# parallel tests introduce random order of requests, can only run live
# Arrange
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = self.get_random_bytes(LARGE_BLOB_SIZE)
FILE_PATH = 'blob_from_stream_chunked_upload.temp.{}.dat'.format(str(uuid.uuid4()))
with open(FILE_PATH, 'wb') as stream:
stream.write(data)
blob_tier = StandardBlobTier.Cool
# Act
content_settings = ContentSettings(
content_type='image/png',
content_language='spanish')
with open(FILE_PATH, 'rb') as stream:
blob.upload_blob(stream, content_settings=content_settings, max_concurrency=2, standard_blob_tier=blob_tier)
properties = blob.get_blob_properties()
# Assert
self.assertEqual(properties.blob_tier, blob_tier)
self._teardown(FILE_PATH)
@GlobalStorageAccountPreparer()
def test_create_blob_from_text(self, resource_group, location, storage_account, storage_account_key):
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
text = u'hello 啊齄丂狛狜 world'
data = text.encode('utf-8')
# Act
create_resp = blob.upload_blob(text)
props = blob.get_blob_properties()
# Assert
self.assertBlobEqual(self.container_name, blob_name, data)
self.assertEqual(props.etag, create_resp.get('etag'))
self.assertEqual(props.last_modified, create_resp.get('last_modified'))
@GlobalStorageAccountPreparer()
def test_create_blob_from_text_with_encoding(self, resource_group, location, storage_account, storage_account_key):
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
text = u'hello 啊齄丂狛狜 world'
data = text.encode('utf-16')
# Act
blob.upload_blob(text, encoding='utf-16')
# Assert
self.assertBlobEqual(self.container_name, blob_name, data)
@GlobalStorageAccountPreparer()
def test_create_blob_from_text_with_encoding_and_progress(self, resource_group, location, storage_account, storage_account_key):
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
text = u'hello 啊齄丂狛狜 world'
data = text.encode('utf-16')
# Act
progress = []
def callback(response):
current = response.context['upload_stream_current']
total = response.context['data_stream_total']
if current is not None:
progress.append((current, total))
blob.upload_blob(text, encoding='utf-16', raw_response_hook=callback)
# Assert
self.assertBlobEqual(self.container_name, blob_name, data)
self.assert_upload_progress(len(data), self.config.max_block_size, progress)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
def test_create_blob_from_text_chunked_upload(self, resource_group, location, storage_account, storage_account_key):
# parallel tests introduce random order of requests, can only run live
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = self.get_random_text_data(LARGE_BLOB_SIZE)
encoded_data = data.encode('utf-8')
# Act
blob.upload_blob(data)
# Assert
self.assertBlobEqual(self.container_name, blob_name, encoded_data)
# Assert
self.assertBlobEqual(self.container_name, blob_name, encoded_data)
@GlobalStorageAccountPreparer()
def test_create_blob_with_md5(self, resource_group, location, storage_account, storage_account_key):
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = b'hello world'
# Act
blob.upload_blob(data, validate_content=True)
# Assert
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
def test_create_blob_with_md5_chunked(self, resource_group, location, storage_account, storage_account_key):
# parallel tests introduce random order of requests, can only run live
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
data = self.get_random_bytes(LARGE_BLOB_SIZE)
# Act
blob.upload_blob(data, validate_content=True)
