def test_set_standard_blob_tier_with_rehydrate_priority(
self, storage_account_name, storage_account_key):
# Arrange
bsc = BlobServiceClient(self.account_url(storage_account_name, "blob"),
credential=storage_account_key)
self._setup(bsc)
blob_client = self._create_blob(bsc)
blob_tier = StandardBlobTier.Archive
rehydrate_tier = StandardBlobTier.Cool
rehydrate_priority = RehydratePriority.standard
# Act
blob_client.set_standard_blob_tier(
blob_tier, rehydrate_priority=rehydrate_priority)
blob_client.set_standard_blob_tier(rehydrate_tier)
blob_props = blob_client.get_blob_properties()
# Assert
self.assertEqual('rehydrate-pending-to-cool',
blob_props.archive_status)
def test_ors_destination(self, resource_group, location, storage_account,
storage_account_key):
# Arrange
bsc = BlobServiceClient(self.account_url(storage_account, "blob"),
credential=storage_account_key)
blob = bsc.get_blob_client(container=self.DST_CONTAINER,
blob=self.BLOB_NAME)
# Act
props = blob.get_blob_properties()
# Assert
self.assertIsInstance(props, BlobProperties)
self.assertIsNotNone(props.object_replication_destination_policy)
# Check that the download function gives back the same result
stream = blob.download_blob()
self.assertEqual(
stream.properties.object_replication_destination_policy,
props.object_replication_destination_policy)
def _get_service_client(self):
if self.connection_string is not None:
return BlobServiceClient.from_connection_string(
self.connection_string)
account_domain = self.custom_domain or "{}.blob.core.windows.net".format(
self.account_name)
account_url = "{}://{}".format(self.azure_protocol, account_domain)
credential = None
if self.account_key:
credential = {
"account_name": self.account_name,
"account_key": self.account_key,
}
elif self.sas_token:
credential = self.sas_token
elif self.token_credential:
credential = self.token_credential
return BlobServiceClient(account_url, credential=credential)
def auth_default_azure_credential(self):
# [START create_blob_service_client_oauth]
# Get a credential for authentication
# Default Azure Credentials attempt a chained set of authentication methods, per documentation here: https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/identity/azure-identity
# For example user (who must be an Azure Event Hubs Data Owner role) to be logged in can be specified by the environment variable AZURE_USERNAME
# Alternately, one can specify the AZURE_TENANT_ID, AZURE_CLIENT_ID, and AZURE_CLIENT_SECRET to use the EnvironmentCredentialClass.
# The docs above specify all mechanisms which the defaultCredential internally support.
from azure.identity import DefaultAzureCredential
default_credential = DefaultAzureCredential()
# Instantiate a BlobServiceClient using a token credential
from azure.storage.blob import BlobServiceClient
blob_service_client = BlobServiceClient(
account_url=self.oauth_url,
credential=default_credential
)
# [END create_blob_service_client_oauth]
# Get account information for the Blob Service
account_info = blob_service_client.get_service_properties()
def main(args,config):
"""
fonction centrale du projet qui permet d'appeler les autres fonctions et leur paramètre avec le parser
"""
blobclient=BlobServiceClient(
f"https://{config['storage']['account']}.blob.core.windows.net",
config["storage"]["key"],
logging_enable=False)
containerclient=blobclient.get_container_client(config["storage"]["container"])
if args.action=="list":
return listb(containerclient)
else:
if args.action=="upload":
blobclient=containerclient.get_blob_client(os.path.basename(args.cible))
return upload(args.cible, blobclient)
elif args.action=="download":
blobclient=containerclient.get_blob_client(os.path.basename(args.remote))
return download(args.remote, config["general"]["restoredir"], blobclient)
elif args.action=="token":
create_token_sas()
def index():
form = UploadForm()
if form.validate_on_submit():
ws_token = form.workspace_token.data.strip()
f = form.file_input.data
filename = secure_filename(f.filename)
str_split=ws_token.split('/')
sas=str_split[3].split('?')[1]
container_name=str_split[3].split('?')[0]
account_url=str_split[0]+'//'+str_split[2]
blob_service_client = BlobServiceClient(account_url=account_url, credential=sas)
container_client = blob_service_client.get_container_client(container_name)
container_client.upload_blob(name=filename,data=f)
flash(f'The file has been uploaded: {filename}')
return redirect(url_for('index'))
return render_template('index.html', title='Upload a file', form=form)
def setUp(self):
super(StorageLargeBlockBlobTest, self).setUp()
url = self._get_account_url()
credential = self._get_shared_key_credential()
# test chunking functionality by reducing the threshold
# for chunking and the size of each chunk, otherwise
# the tests would take too long to execute
self.bsc = BlobServiceClient(url,
credential=credential,
max_single_put_size=32 * 1024,
max_block_size=2 * 1024 * 1024,
min_large_block_upload_threshold=1 *
1024 * 1024)
self.config = self.bsc._config
self.container_name = self.get_resource_name('utcontainer')
if not self.is_playback():
self.bsc.create_container(self.container_name)
def save_to_blob(self,
connection_string: str,
container: str,
path: str = None) -> str:
""" Saves the model to an azure storage blob.
The file will be located at the container and path parameter if provided, otherwise, in the root of the container."""
path = path or 'azureSDKTrackClassifier_{}_{}.model'.format(
self._language, self._service)
if 'sig=' in connection_string and 'AccountKey=' not in connection_string: # SAS signature.
service_client = BlobServiceClient(connection_string)
else:
service_client = BlobServiceClient.from_connection_string(
conn_str=connection_string)
try:
service_client.create_container(container)
except ResourceExistsError:
pass
blob_client = service_client.get_blob_client(container, path)
blob_client.upload_blob(pickle.dumps(self), overwrite=True)
return path
def test_create_append_blob_with_chunks(self, resource_group, location,
storage_account,
storage_account_key):
# Arrange
# test chunking functionality by reducing the size of each chunk,
# otherwise the tests would take too long to execute
bsc = BlobServiceClient(self._account_url(storage_account.name),
credential=storage_account_key,
connection_data_block_size=1024,
max_single_put_size=1024,
min_large_block_upload_threshold=1024,
max_block_size=1024,
max_page_size=1024)
self._setup(bsc)
blob_client = self._create_append_blob(bsc, cpk=TEST_ENCRYPTION_KEY)
# Act
append_blob_prop = blob_client.upload_blob(
self.byte_data,
blob_type=BlobType.AppendBlob,
cpk=TEST_ENCRYPTION_KEY)
# Assert
self.assertIsNotNone(append_blob_prop['etag'])
self.assertIsNotNone(append_blob_prop['last_modified'])
self.assertTrue(append_blob_prop['request_server_encrypted'])
self.assertEqual(append_blob_prop['encryption_key_sha256'],
TEST_ENCRYPTION_KEY.key_hash)
# Act get the blob content without cpk should fail
with self.assertRaises(HttpResponseError):
blob_client.download_blob()
# Act get the blob content
blob = blob_client.download_blob(cpk=TEST_ENCRYPTION_KEY)
# Assert content was retrieved with the cpk
self.assertEqual(blob.readall(), self.byte_data)
self.assertEqual(blob.properties.encryption_key_sha256,
TEST_ENCRYPTION_KEY.key_hash)
self._teardown(bsc)
def test_quick_query_iter_records_with_nonfatal_error_handler(self, storage_account_name, storage_account_key):
# Arrange
bsc = BlobServiceClient(
self.account_url(storage_account_name, "blob"),
credential=storage_account_key)
self._setup(bsc)
# upload the csv file
blob_name = self._get_blob_reference()
blob_client = bsc.get_blob_client(self.container_name, blob_name)
blob_client.upload_blob(CSV_DATA, overwrite=True)
errors = []
def on_error(error):
errors.append(error)
input_format = DelimitedTextDialect(
delimiter=',',
quotechar='"',
lineterminator='\n',
escapechar='',
has_header=True
)
output_format = DelimitedTextDialect(
delimiter=';',
quotechar="'",
lineterminator='%',
escapechar='\\',
)
resp = blob_client.query_blob(
"SELECT RepoPath from BlobStorage",
blob_format=input_format,
output_format=output_format,
on_error=on_error)
data = list(resp.records())
# the error is because that line only has one column
self.assertEqual(len(errors), 1)
self.assertEqual(resp._size, len(CSV_DATA))
self.assertEqual(len(data), 32)
self._teardown(bsc)
def test_quick_query_readall_with_serialization_setting(self, storage_account_name, storage_account_key):
# Arrange
bsc = BlobServiceClient(
self.account_url(storage_account_name, "blob"),
credential=storage_account_key)
self._setup(bsc)
# upload the csv file
blob_name = self._get_blob_reference()
blob_client = bsc.get_blob_client(self.container_name, blob_name)
blob_client.upload_blob(CSV_DATA, overwrite=True)
errors = []
def on_error(error):
errors.append(error)
input_format = DelimitedTextDialect(
delimiter=',',
quotechar='"',
lineterminator='\n',
escapechar='',
has_header=False
)
output_format = DelimitedTextDialect(
delimiter=';',
quotechar="'",
lineterminator='.',
escapechar='\\'
)
resp = blob_client.query_blob(
"SELECT * from BlobStorage",
on_error=on_error,
blob_format=input_format,
output_format=output_format)
query_result = resp.readall()
self.assertEqual(len(errors), 0)
self.assertEqual(resp._size, len(CSV_DATA))
self.assertEqual(query_result, CONVERTED_CSV_DATA)
self._teardown(bsc)
def test_get_set_blob_metadata(self, resource_group, location,
storage_account, storage_account_key):
# Arrange
# test chunking functionality by reducing the size of each chunk,
# otherwise the tests would take too long to execute
bsc = BlobServiceClient(self.account_url(storage_account, "blob"),
credential=storage_account_key,
connection_data_block_size=1024,
max_single_put_size=1024,
min_large_block_upload_threshold=1024,
max_block_size=1024,
max_page_size=1024)
self._setup(bsc)
blob_client, _ = self._create_block_blob(
bsc, data=b'AAABBBCCC', encryption_scope=TEST_ENCRYPTION_KEY_SCOPE)
# Act
blob_props = blob_client.get_blob_properties()
# Assert
self.assertTrue(blob_props.server_encrypted)
self.assertEqual(blob_props['encryption_scope'],
TEST_ENCRYPTION_KEY_SCOPE)
# Act set blob properties
metadata = {'hello': 'world', 'number': '42', 'up': 'upval'}
with self.assertRaises(HttpResponseError):
blob_client.set_blob_metadata(metadata=metadata, )
blob_client.set_blob_metadata(
metadata=metadata, encryption_scope=TEST_ENCRYPTION_KEY_SCOPE)
# Assert
blob_props = blob_client.get_blob_properties()
md = blob_props.metadata
self.assertEqual(3, len(md))
self.assertEqual(md['hello'], 'world')
self.assertEqual(md['number'], '42')
self.assertEqual(md['up'], 'upval')
self.assertFalse('Up' in md)
self._teardown(bsc)
def test_create_block_blob_with_chunks(self, resource_group, location, storage_account, storage_account_key):
# parallel operation
# test chunking functionality by reducing the size of each chunk,
# otherwise the tests would take too long to execute
bsc = BlobServiceClient(
self.account_url(storage_account.name, "blob"),
credential=storage_account_key,
connection_data_block_size=1024,
max_single_put_size=1024,
min_large_block_upload_threshold=1024,
max_block_size=1024,
max_page_size=1024)
self._setup(bsc)
# Arrange
# to force the in-memory chunks to be used
self.config.use_byte_buffer = True
# Act
# create_blob_from_bytes forces the in-memory chunks to be used
blob_client, upload_response = self._create_block_blob(bsc, data=self.byte_data, cpk=TEST_ENCRYPTION_KEY,
max_concurrency=2)
# Assert
self.assertIsNotNone(upload_response['etag'])
self.assertIsNotNone(upload_response['last_modified'])
self.assertTrue(upload_response['request_server_encrypted'])
self.assertEqual(upload_response['encryption_key_sha256'], TEST_ENCRYPTION_KEY.key_hash)
# Act get the blob content without cpk should fail
with self.assertRaises(HttpResponseError):
blob_client.download_blob()
# Act get the blob content
blob = blob_client.download_blob(cpk=TEST_ENCRYPTION_KEY)
# Assert content was retrieved with the cpk
self.assertEqual(blob.readall(), self.byte_data)
self.assertEqual(blob.properties.etag, upload_response['etag'])
self.assertEqual(blob.properties.last_modified, upload_response['last_modified'])
self.assertEqual(blob.properties.encryption_key_sha256, TEST_ENCRYPTION_KEY.key_hash)
self._teardown(bsc)
def main(mytimer: func.TimerRequest) -> None:
storageName = os.getenv("STORAGE_ACCOUNT_NAME")
containerName = os.getenv("BLOB_CONTAINER_NAME")
logging.info(f"params: {storageName}, {containerName}")
ts = datetime.datetime.now()
iso_ts = ts.isoformat()
date_str = ts.strftime("%Y%m%d")
credential = DefaultAzureCredential()
oauth_url = f"https://{storageName}.blob.core.windows.net"
bsc = BlobServiceClient(account_url=oauth_url, credential=credential)
containerClient = bsc.get_container_client(containerName)
tickers = ["BTC-USD", "MSFT"]
n = len(tickers)
for i in range(n):
block = ",\n"
ticker = tickers[i]
logging.info(f"storing {ticker}")
blobName = createBlobName(ticker, date_str)
logging.info(f"container: {containerName}, blob: {blobName}")
blobClient = containerClient.get_blob_client(blobName)
try:
if not blobClient.exists():
blobClient.create_append_blob()
block = ""
except ResourceNotFoundError:
logging.info(f"Creating container: {containerName}")
containerClient = bsc.create_container(containerName)
blobClient = containerClient.get_blob_client(blobName)
blobClient.create_append_blob()
block += json.dumps({"ts": iso_ts, "v": getCurrClose(ticker)})
blobClient.append_block(block)
if i<n-1: time.sleep(1)
def enumerate_folders():
#%% Derived constants
storage_account_url_blob = 'https://' + account_name + '.blob.core.windows.net'
#%% Create client handle
blob_service_client = BlobServiceClient(
account_url=storage_account_url_blob, credential=ro_sas_token)
container_client = blob_service_client.get_container_client(container_name)
#%% Enumerate row-level folders
start = datetime.datetime.now()
#
# Uses ContainerClient.walk_blobs()
#
folders, _ = walk_container(container_client,
max_depth=depth,
store_blobs=False)
end = datetime.datetime.now()
elapsed = end - start
folders = [s for s in folders if s.count('/') == (depth - 1)]
print("Enumerated {} folders in {}s".format(len(folders),
str(elapsed.seconds)))
for s in folders:
print(s)
#%% Write results to file
folders_with_newlines = [s + '\n' for s in folders]
with open(output_file, 'w') as f:
f.writelines(folders_with_newlines)
def upload_third_party(self) -> None:
logger.info("uploading third-party tools from %s", self.third_party)
account_name = self.results["deploy"]["fuzz-name"]["value"]
key = self.results["deploy"]["fuzz-key"]["value"]
account_url = "https://%s.blob.core.windows.net" % account_name
client = BlobServiceClient(account_url, credential=key)
containers = [x["name"] for x in client.list_containers()]
for name in os.listdir(self.third_party):
path = os.path.join(self.third_party, name)
if not os.path.isdir(path):
continue
if name not in containers:
client.create_container(name)
expiry = datetime.utcnow() + timedelta(minutes=30)
sas = generate_container_sas(
account_name,
name,
account_key=key,
permission=ContainerSasPermissions(read=True,
write=True,
delete=True,
list=True),
expiry=expiry,
)
url = "%s/%s?%s" % (account_url, name, sas)
subprocess.check_output([
self.azcopy,
"copy",
os.path.join(path, "*"),
url,
"--overwrite=true",
"--recursive=true",
])
subprocess.check_output([
self.azcopy, "sync", path, url, "--delete-destination", "true"
])
def test_quick_query_iter_records_with_fatal_error_handler_raise(
self, resource_group, location, storage_account,
storage_account_key):
# Arrange
bsc = BlobServiceClient(self.account_url(storage_account, "blob"),
credential=storage_account_key)
self._setup(bsc)
data1 = b'{name: owner}'
data2 = b'{name2: owner2}'
data3 = b'{version:0,begin:1601-01-01T00:00:00.000Z,intervalSecs:3600,status:Finalized,config:' \
b'{version:0,configVersionEtag:0x8d75ef460eb1a12,numShards:1,recordsFormat:avro,formatSchemaVersion:3,' \
b'shardDistFnVersion:1},chunkFilePaths:[$blobchangefeed/log/00/1601/01/01/0000/],storageDiagnostics:' \
b'{version:0,lastModifiedTime:2019-11-01T17:53:18.861Z,' \
b'data:{aid:d305317d-a006-0042-00dd-902bbb06fc56}}}'
data = data1 + b'\n' + data2 + b'\n' + data1
# upload the json file
blob_name = self._get_blob_reference()
blob_client = bsc.get_blob_client(self.container_name, blob_name)
blob_client.upload_blob(data, overwrite=True)
errors = []
def on_error(error):
raise Exception(error.description)
input_format = DelimitedJsonDialect()
output_format = DelimitedTextDialect(delimiter=';',
quotechar="'",
lineterminator='.',
escapechar='\\')
resp = blob_client.query_blob("SELECT * from BlobStorage",
on_error=on_error,
blob_format=input_format,
output_format=output_format)
with pytest.raises(Exception):
for record in resp.records():
print(record)
self._teardown(bsc)
def get_conn(self) -> BlobServiceClient: # pylint: disable=too-many-return-statements
"""Return the BlobServiceClient object."""
conn = self.get_connection(self.conn_id)
extra = conn.extra_dejson or {}
if self.public_read:
# Here we use anonymous public read
# more info
# https://docs.microsoft.com/en-us/azure/storage/blobs/storage-manage-access-to-resources
return BlobServiceClient(account_url=conn.host)
if extra.get('connection_string') or extra.get(
'extra__wasb__connection_string'):
# connection_string auth takes priority
connection_string = extra.get('connection_string') or extra.get(
'extra__wasb__connection_string')
return BlobServiceClient.from_connection_string(connection_string)
if extra.get('shared_access_key') or extra.get(
'extra__wasb__shared_access_key'):
shared_access_key = extra.get('shared_access_key') or extra.get(
'extra__wasb__shared_access_key')
# using shared access key
return BlobServiceClient(account_url=conn.host,
credential=shared_access_key)
if extra.get('tenant_id') or extra.get('extra__wasb__tenant_id'):
# use Active Directory auth
app_id = conn.login
app_secret = conn.password
tenant = extra.get('tenant_id') or extra.get(
'extra__wasb__tenant_id')
token_credential = ClientSecretCredential(tenant, app_id,
app_secret)
return BlobServiceClient(account_url=conn.host,
credential=token_credential)
sas_token = extra.get('sas_token') or extra.get(
'extra__wasb__sas_token')
if sas_token and sas_token.startswith('https'):
return BlobServiceClient(account_url=extra.get('sas_token'))
if sas_token and not sas_token.startswith('https'):
return BlobServiceClient(
account_url=f"https://{conn.login}.blob.core.windows.net/" +
sas_token)
else:
# Fall back to old auth
return BlobServiceClient(
account_url=f"https://{conn.login}.blob.core.windows.net/",
credential=conn.password,
**extra)
def test_quick_query_iter_records_with_encoding(self, resource_group, location, storage_account, storage_account_key):
# Arrange
bsc = BlobServiceClient(
self.account_url(storage_account, "blob"),
credential=storage_account_key)
self._setup(bsc)
# upload the csv file
blob_name = self._get_blob_reference()
blob_client = bsc.get_blob_client(self.container_name, blob_name)
blob_client.upload_blob(CSV_DATA, overwrite=True)
reader = blob_client.query_blob("SELECT * from BlobStorage", encoding='utf-8')
data = ''
for record in reader.records():
data += record
self.assertEqual(len(reader), len(CSV_DATA))
self.assertEqual(reader._size, reader._bytes_processed)
self.assertEqual(data, CSV_DATA.replace(b'\r\n', b'').decode('utf-8'))
self._teardown(bsc)
def __init__(self, *args, **kwargs):
super(DatabaseWrapper, self).__init__(*args, **kwargs)
# token_credential = os.getenv("AZURE_STORAGE_CONNECTION_STRING")
# blob_service_client = BlobServiceClient.from_connection_string(self.connection_string)
# the answer is somewhere here
# https://github.com/Azure/azure-sdk-for-python/blob/azure-storage-blob_12.8.1/sdk/storage/azure-storage-blob/samples/blob_samples_hello_world.py
# DefaultAzureCredential should use:
# AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_CLIENT_SECRET
token_credential = DefaultAzureCredential()
bucket = self.settings_dict["AZ_BLOB_BUCKET"] # yourname.blob.core.windows.net/
self.az = BlobServiceClient(account_url=f"https://${self.settings_dict['BUCKET']}", credential=token_credential)
#signature_version = self.settings_dict.get("SIGNATURE_VERSION", "s3v4")
#self.s3 = boto3.resource(
# "s3", config=botocore.client.Config(signature_version=signature_version),
#)
self.db_hash = None
self.load_remote_db()
def __init__(self, artifact_uri, client=None):
super(AzureBlobArtifactRepository, self).__init__(artifact_uri)
# Allow override for testing
if client:
self.client = client
return
from azure.storage.blob import BlobServiceClient
(_, account, _) = AzureBlobArtifactRepository.parse_wasbs_uri(artifact_uri)
if "AZURE_STORAGE_CONNECTION_STRING" in os.environ:
self.client = BlobServiceClient.from_connection_string(
conn_str=os.environ.get("AZURE_STORAGE_CONNECTION_STRING"))
elif "AZURE_STORAGE_ACCESS_KEY" in os.environ:
account_url = "https://{account}.blob.core.windows.net".format(account=account)
self.client = BlobServiceClient(
account_url=account_url,
credential=os.environ.get("AZURE_STORAGE_ACCESS_KEY"))
else:
raise Exception("You need to set one of AZURE_STORAGE_CONNECTION_STRING or "
"AZURE_STORAGE_ACCESS_KEY to access Azure storage.")
async def uploadFile(data, originalFileName: str):
try:
lastSplitIndex = originalFileName.rindex(".")
fileName = originalFileName[0:lastSplitIndex]
fileExtension = originalFileName.split(".").pop()
blobName = fileName + str(int(round(
time.time() * 1000))) + '.' + fileExtension
blobServiceClient = BlobServiceClient(
account_url=FILE_STORAGE_URL, credential=FILE_STORAGE_UPLOAD_KEY)
containerClient = blobServiceClient.get_container_client(
FILE_STORAGE_DIRECTORY)
blockBlobClient = containerClient.get_blob_client(blobName)
blockBlobClient.upload_blob(data)
url = urlparse(blockBlobClient.url)
return {
"Name": blobName,
"Url": url.scheme + "://" + url.netloc + url.path
}
except Exception as e:
print(e)
def get_ooiopendata_blobs(container=None, sas_token=None):
"""
Return a list of blob properties from a container in the ooiopendata storage account.
Parameters
----------
container : str
Name of container.
sas_token : str
SAS token for authenticated access.
Returns
-------
list : Azure BlobProperties class.
"""
storage_account_url = 'https://ooiopendata.blob.core.windows.net'
blob_service_client = BlobServiceClient(storage_account_url,
credential=sas_token)
container_client = blob_service_client.get_container_client(container)
ooiopendata_blobs = [blob for blob in container_client.list_blobs()]
return ooiopendata_blobs
def test_request_callback_signed_header(self, resource_group, location,
storage_account,
storage_account_key):
# Arrange
service = BlobServiceClient(self.account_url(storage_account, "blob"),
credential=storage_account_key)
name = self.get_resource_name('cont')
# Act
def callback(request):
if request.http_request.method == 'PUT':
request.http_request.headers['x-ms-meta-hello'] = 'world'
# Assert
try:
container = service.create_container(name,
raw_request_hook=callback)
metadata = container.get_container_properties().metadata
self.assertEqual(metadata, {'hello': 'world'})
finally:
service.delete_container(name)
def test_quick_query_output_in_parquet_format(self, resource_group,
location, storage_account,
storage_account_key):
# Arrange
bsc = BlobServiceClient(self.account_url(storage_account, "blob"),
credential=storage_account_key)
self._setup(bsc)
expression = "SELECT * from BlobStorage"
blob_name = self._get_blob_reference()
blob_client = bsc.get_blob_client(self.container_name, blob_name)
parquet_path = os.path.abspath(
os.path.join(os.path.abspath(__file__), "..",
"./resources/parquet.parquet"))
with open(parquet_path, "rb") as parquet_data:
blob_client.upload_blob(parquet_data, overwrite=True)
with self.assertRaises(ValueError):
blob_client.query_blob(expression,
blob_format="ParquetDialect",
output_format="ParquetDialect")
def test_copy_source_sas_is_scrubbed_off(self, resource_group, location,
storage_account,
storage_account_key):
# SAS URL is calculated from storage key, so this test runs live only
bsc = BlobServiceClient(self.account_url(storage_account, "blob"),
storage_account_key)
self._setup(bsc)
# Arrange
dest_blob_name = self.get_resource_name('destblob')
dest_blob = bsc.get_blob_client(self.container_name, dest_blob_name)
# parse out the signed signature
query_parameters = urlparse(self.source_blob_url).query
token_components = parse_qs(query_parameters)
if QueryStringConstants.SIGNED_SIGNATURE not in token_components:
pytest.fail(
"Blob URL {} doesn't contain {}, parsed query params: {}".
format(self.source_blob_url,
QueryStringConstants.SIGNED_SIGNATURE,
list(token_components.keys())))
signed_signature = quote(
token_components[QueryStringConstants.SIGNED_SIGNATURE][0])
# Act
with LogCaptured(self) as log_captured:
dest_blob.start_copy_from_url(self.source_blob_url,
requires_sync=True,
logging_enable=True)
log_as_str = log_captured.getvalue()
# Assert
# make sure the query parameter 'sig' is logged, but its value is not
self.assertTrue(
QueryStringConstants.SIGNED_SIGNATURE in log_as_str)
self.assertFalse(signed_signature in log_as_str)
# make sure authorization header is logged, but its value is not
# the keyword SharedKey is present in the authorization header's value
self.assertTrue(_AUTHORIZATION_HEADER_NAME in log_as_str)
self.assertFalse('SharedKey' in log_as_str)
def test_create_container_with_default_cpk_n(self, resource_group,
location, storage_account,
storage_account_key):
# Arrange
bsc = BlobServiceClient(self.account_url(storage_account, "blob"),
credential=storage_account_key,
connection_data_block_size=1024,
max_single_put_size=1024,
min_large_block_upload_threshold=1024,
max_block_size=1024,
max_page_size=1024)
container_client = bsc.create_container(
'cpkcontainer',
container_encryption_scope=TEST_CONTAINER_ENCRYPTION_KEY_SCOPE)
container_props = container_client.get_container_properties()
self.assertEqual(
container_props.encryption_scope.default_encryption_scope,
TEST_CONTAINER_ENCRYPTION_KEY_SCOPE.default_encryption_scope)
self.assertEqual(
container_props.encryption_scope.prevent_encryption_scope_override,
False)
for container in bsc.list_containers(name_starts_with='cpkcontainer'):
self.assertEqual(
container_props.encryption_scope.default_encryption_scope,
TEST_CONTAINER_ENCRYPTION_KEY_SCOPE.default_encryption_scope)
self.assertEqual(
container_props.encryption_scope.
prevent_encryption_scope_override, False)
blob_client = container_client.get_blob_client("appendblob")
# providing encryption scope when upload the blob
resp = blob_client.upload_blob(
b'aaaa',
BlobType.AppendBlob,
encryption_scope=TEST_ENCRYPTION_KEY_SCOPE)
# Use the provided encryption scope on the blob
self.assertEqual(resp['encryption_scope'], TEST_ENCRYPTION_KEY_SCOPE)
container_client.delete_container()
def test_set_static_website_properties_missing_field(
self, resource_group, location, storage_account,
storage_account_key):
bsc = BlobServiceClient(self._account_url(storage_account.name),
credential=storage_account_key)
# Case1: Arrange both missing
static_website = StaticWebsite(enabled=True)
# Act
bsc.set_service_properties(static_website=static_website)
# Assert
received_props = bsc.get_service_properties()
self._assert_static_website_equal(received_props['static_website'],
static_website)
# Case2: Arrange index document missing
static_website = StaticWebsite(
enabled=True, error_document404_path="errors/error/404error.html")
# Act
bsc.set_service_properties(static_website=static_website)
# Assert
received_props = bsc.get_service_properties()
self._assert_static_website_equal(received_props['static_website'],
static_website)
# Case3: Arrange error document missing
static_website = StaticWebsite(enabled=True,
index_document="index.html")
# Act
bsc.set_service_properties(static_website=static_website)
# Assert
received_props = bsc.get_service_properties()
self._assert_static_website_equal(received_props['static_website'],
static_website)
def __init__(self, *args, **kwargs):
if "stay_on_remote" in kwargs:
del kwargs["stay_on_remote"]
# if not handed down explicitely, try to read credentials from
# environment variables.
for (csavar, envvar) in [
("account_url", "AZ_BLOB_ACCOUNT_URL"),
("credential", "AZ_BLOB_CREDENTIAL"),
]:
if csavar not in kwargs and envvar in os.environ:
kwargs[csavar] = os.environ.get(envvar)
assert (
"account_url" in kwargs
), "Missing AZ_BLOB_ACCOUNT_URL env var (and possibly AZ_BLOB_CREDENTIAL)"
# remove leading '?' from SAS if needed
# if kwargs.get("sas_token", "").startswith("?"):
# kwargs["sas_token"] = kwargs["sas_token"][1:]
# by right only account_key or sas_token should be set, but we let
# BlobServiceClient deal with the ambiguity
self.blob_service_client = BlobServiceClient(**kwargs)
def create_client(stage_info, use_accelerate_endpoint: bool = False):
"""Creates a client object with a stage credential.
Args:
stage_info: Information about the stage.
use_accelerate_endpoint: Not used for Azure client.
Returns:
The client to communicate with GCS.
"""
stage_credentials = stage_info['creds']
sas_token = stage_credentials['AZURE_SAS_TOKEN']
if sas_token and sas_token.startswith('?'):
sas_token = sas_token[1:]
end_point = stage_info['endPoint']
if end_point.startswith('blob.'):
end_point = end_point[len('blob.'):]
if use_new_azure_api:
client = BlobServiceClient(account_url="https://{}.blob.{}".format(
stage_info['storageAccount'], end_point),
credential=sas_token)
client._config.retry_policy = ExponentialRetry(
initial_backoff=1,
increment_base=2,
max_attempts=60,
random_jitter_range=2)
else:
client = BlockBlobService(
account_name=stage_info['storageAccount'],
sas_token=sas_token,
endpoint_suffix=end_point)
client._httpclient = RawBodyReadingClient(
session=requests.session(), protocol="https", timeout=2000)
client.retry = ExponentialRetry(initial_backoff=1,
increment_base=2,
max_attempts=60,
random_jitter_range=2).retry
return client
