def main(req: func.HttpRequest) -> func.HttpResponse:
logging.info(req.get_json())
try:
is_pepper_include = req.get_json()['pepper']
except:
return func.HttpResponse("Write down whether to include pepper or not",
status_code=400)
try:
if is_pepper_include:
url = os.environ["PEPPER_CONTAINER"]
container = ContainerClient.from_container_url(url)
else:
url = os.environ["NO_PEPPER_CONTAINER"]
container = ContainerClient.from_container_url(url)
blob_list = list(container.list_blobs())
blob = choice(blob_list)
except Exception as e:
logging.info("Err: {}".format(e))
return func.HttpResponse(str(e), status_code=500)
image_url = blob.name
return_data = {"tteokgukImageUrl": url + image_url}
return func.HttpResponse(json.dumps(return_data),
headers={"Content-Type": "application/json"},
status_code=200)
def test_create_client_for_emulator(self):
container_client = ContainerClient(
account_url='http://127.0.0.1:1000/devstoreaccount1',
container_name='newcontainer',
credential=
'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=='
)
self.assertEqual(container_client.container_name, "newcontainer")
self.assertEqual(container_client.account_name, "devstoreaccount1")
ContainerClient.from_container_url(
'http://127.0.0.1:1000/devstoreaccount1/newcontainer')
self.assertEqual(container_client.container_name, "newcontainer")
self.assertEqual(container_client.account_name, "devstoreaccount1")
def test_sas_signature_is_scrubbed_off(self, resource_group, location,
storage_account,
storage_account_key):
# SAS URL is calculated from storage key, so this test runs live only
bsc = BlobServiceClient(self.account_url(storage_account, "blob"),
storage_account_key)
self._setup(bsc)
# Arrange
container = bsc.get_container_client(self.container_name)
token = generate_container_sas(
container.account_name,
container.container_name,
account_key=container.credential.account_key,
permission=ContainerSasPermissions(read=True),
expiry=datetime.utcnow() + timedelta(hours=1),
)
# parse out the signed signature
token_components = parse_qs(token)
signed_signature = quote(
token_components[QueryStringConstants.SIGNED_SIGNATURE][0])
sas_service = ContainerClient.from_container_url(container.url,
credential=token)
# Act
with LogCaptured(self) as log_captured:
sas_service.get_account_information(logging_enable=True)
log_as_str = log_captured.getvalue()
# Assert
# make sure the query parameter 'sig' is logged, but its value is not
self.assertTrue(
QueryStringConstants.SIGNED_SIGNATURE in log_as_str)
self.assertFalse(signed_signature in log_as_str)
def container_access_policy(self):
# SAS URL is calculated from storage key, so this test runs live only
if TestMode.need_recording_file(self.test_mode):
return
# Instantiate a BlobServiceClient using a connection string
from azure.storage.blob import BlobServiceClient
blob_service_client = BlobServiceClient.from_connection_string(
self.connection_string)
# Instantiate a ContainerClient
container_client = blob_service_client.get_container_client(
"myaccesscontainer")
try:
# Create new Container
container_client.create_container()
# [START set_container_access_policy]
# Create access policy
from azure.storage.blob import AccessPolicy, ContainerSasPermissions
access_policy = AccessPolicy(
permission=ContainerSasPermissions(read=True),
expiry=datetime.utcnow() + timedelta(hours=1),
start=datetime.utcnow() - timedelta(minutes=1))
identifiers = {'test': access_policy}
# Set the access policy on the container
container_client.set_container_access_policy(
signed_identifiers=identifiers)
# [END set_container_access_policy]
# [START get_container_access_policy]
policy = container_client.get_container_access_policy()
# [END get_container_access_policy]
# [START generate_sas_token]
# Use access policy to generate a sas token
from azure.storage.blob import generate_container_sas
sas_token = generate_container_sas(
container_client.account_name,
container_client.container_name,
account_key=container_client.credential.account_key,
policy_id='my-access-policy-id')
# [END generate_sas_token]
# Use the sas token to authenticate a new client
# [START create_container_client_sastoken]
from azure.storage.blob import ContainerClient
container = ContainerClient.from_container_url(
container_url=
"https://account.blob.core.windows.net/mycontainer",
credential=sas_token)
# [END create_container_client_sastoken]
finally:
# Delete container
container_client.delete_container()
def test_sas_signature_is_scrubbed_off(self):
# SAS URL is calculated from storage key, so this test runs live only
if TestMode.need_recording_file(self.test_mode):
return
# Arrange
container = self.bsc.get_container_client(self.container_name)
token = container.generate_shared_access_signature(
permission=ContainerSasPermissions(read=True),
expiry=datetime.utcnow() + timedelta(hours=1),
)
# parse out the signed signature
token_components = parse_qs(token)
signed_signature = quote(
token_components[QueryStringConstants.SIGNED_SIGNATURE][0])
sas_service = ContainerClient.from_container_url(container.url,
credential=token)
# Act
with LogCaptured(self) as log_captured:
sas_service.get_account_information(logging_enable=True)
log_as_str = log_captured.getvalue()
# Assert
# make sure the query parameter 'sig' is logged, but its value is not
self.assertTrue(
QueryStringConstants.SIGNED_SIGNATURE in log_as_str)
self.assertFalse(signed_signature in log_as_str)
def enumerate_files(dir: str, ext: str, credentials: Optional[Union[str,Dict[str,str]]]):
blob_data = _try_parse_azure_blob_uri(dir)
if blob_data is None:
return [os.path.join(dir, path.name)
for path in os.scandir(dir)
if path.is_file() and (ext is None or path.name.endswith(ext))]
else:
try:
# pip install azure-storage-blob
from azure.storage.blob import ContainerClient
except:
print("Failed to import azure.storage.blob. Please pip install azure-storage-blob", file=sys.stderr)
raise
account, container, blob_path = blob_data
print("enumerate_files: enumerating blobs in", dir, file=sys.stderr, flush=True)
# @BUGBUG: The prefix does not seem to have to start; seems it can also be a substring
container_uri = "https://" + account + ".blob.core.windows.net/" + container
container_client = ContainerClient.from_container_url(container_uri, credential=_get_azure_key(account, credentials))
if not blob_path.endswith("/"):
blob_path += "/"
blob_uris = [container_uri + "/" + blob["name"] for blob in container_client.walk_blobs(blob_path, delimiter="") if (ext is None or blob["name"].endswith(ext))]
print("enumerate_files:", len(blob_uris), "blobs found", file=sys.stderr, flush=True)
for blob_name in blob_uris[:10]:
print(blob_name, file=sys.stderr, flush=True)
return blob_uris
def CopyImages():
"""Ingest the images to the data lake.
"""
print('Transferring images to Data Lake.')
container = ContainerClient.from_container_url(
container_url="https://bus5wb.blob.core.windows.net/imagecollection",
credential=
"?st=2021-05-19T04%3A43%3A08Z&se=2022-05-20T04%3A43%3A00Z&sp=rl&sv=2018-03-28&sr=c&sig=yV19KD0EzGYQOecRpa2em6Fc6IRQ7%2FhowiAaO%2Bk70O4%3D"
)
container_client = ContainerClient.from_connection_string(
connection_string, container_name=container_name_images)
blobs_list = container.list_blobs()
for blob in blobs_list:
blob_client = container.get_blob_client(blob.name)
nycImageUrl = blob_client.url
# Generate filename for image metadata file.
filePath = urlparse(nycImageUrl)[2]
filePathParts = filePath.split('/')
fileName = filePathParts[len(filePathParts) - 1]
try:
# Instantiate a new BlobClient
blob_client = container_client.get_blob_client(fileName)
# Upload the blob
blob_client.start_copy_from_url(nycImageUrl)
except Exception as e:
print(f"Error {e} - {fileName} - {nycImageUrl}")
def container_sample(self):
# [START create_container_client_from_service]
# Instantiate a BlobServiceClient using a connection string
from azure.storage.blob import BlobServiceClient
blob_service_client = BlobServiceClient.from_connection_string(
self.connection_string)
# Instantiate a ContainerClient
container_client = blob_service_client.get_container_client(
"mynewcontainer")
# [END create_container_client_from_service]
# [START create_container_client_sasurl]
from azure.storage.blob import ContainerClient
sas_url = "https://account.blob.core.windows.net/mycontainer?sv=2015-04-05&st=2015-04-29T22%3A18%3A26Z&se=2015-04-30T02%3A23%3A26Z&sr=b&sp=rw&sip=168.1.5.60-168.1.5.70&spr=https&sig=Z%2FRHIX5Xcg0Mq2rqI3OlWTjEg2tYkboXr1P9ZUXDtkk%3D"
container = ContainerClient.from_container_url(sas_url)
# [END create_container_client_sasurl]
try:
# [START create_container]
container_client.create_container()
# [END create_container]
# [START get_container_properties]
properties = container_client.get_container_properties()
# [END get_container_properties]
finally:
# [START delete_container]
container_client.delete_container()
def create_container_client(self, account, container):
if self.connection_string:
client = ContainerClient.from_connection_string(
self.connection_string, container)
else:
client = ContainerClient.from_container_url("https://{}/{}".format(
account, container))
return client
def get_container_client(self, hex_obj_id):
"""Get the container client for the container that contains the object with
internal id hex_obj_id
This is used to allow the PrefixedAzureCloudObjStorage to dispatch the
client according to the prefix of the object id.
"""
return ContainerClient.from_container_url(self.container_url)
def get_blob_sas_uri(self, container_sas_uri, blob_name):
container_client = ContainerClient.from_container_url(
container_sas_uri)
blob_client = container_client.get_blob_client(blob_name)
account_name = self.get_account_from_uri(container_sas_uri)
container_name = self.get_container_from_uri(container_sas_uri)
sas_key = self.get_sas_key_from_uri(container_sas_uri)
return 'https://{}.blob.core.windows.net/{}/{}?{}'.format(
account_name, container_name, blob_name, sas_key)
def test_create_service_with_custom_account_endpoint_path(self):
account_name = "blobstorage"
account_key = "blobkey"
custom_account_url = "http://local-machine:11002/custom/account/path/" + self.sas_token
for service_type in SERVICES.items():
conn_string = 'DefaultEndpointsProtocol=http;AccountName={};AccountKey={};BlobEndpoint={};'.format(
account_name, account_key, custom_account_url)
# Act
service = service_type[0].from_connection_string(
conn_string, container_name="foo", blob_name="bar")
# Assert
self.assertEqual(service.account_name, account_name)
self.assertEqual(service.credential.account_name, account_name)
self.assertEqual(service.credential.account_key, account_key)
self.assertEqual(service.primary_hostname, 'local-machine:11002/custom/account/path')
service = BlobServiceClient(account_url=custom_account_url)
self.assertEqual(service.account_name, None)
self.assertEqual(service.credential, None)
self.assertEqual(service.primary_hostname, 'local-machine:11002/custom/account/path')
self.assertTrue(service.url.startswith('http://local-machine:11002/custom/account/path/?'))
service = ContainerClient(account_url=custom_account_url, container_name="foo")
self.assertEqual(service.account_name, None)
self.assertEqual(service.container_name, "foo")
self.assertEqual(service.credential, None)
self.assertEqual(service.primary_hostname, 'local-machine:11002/custom/account/path')
self.assertTrue(service.url.startswith('http://local-machine:11002/custom/account/path/foo?'))
service = ContainerClient.from_container_url("http://local-machine:11002/custom/account/path/foo?query=value")
self.assertEqual(service.account_name, None)
self.assertEqual(service.container_name, "foo")
self.assertEqual(service.credential, None)
self.assertEqual(service.primary_hostname, 'local-machine:11002/custom/account/path')
self.assertEqual(service.url, 'http://local-machine:11002/custom/account/path/foo')
service = BlobClient(account_url=custom_account_url, container_name="foo", blob_name="bar", snapshot="baz")
self.assertEqual(service.account_name, None)
self.assertEqual(service.container_name, "foo")
self.assertEqual(service.blob_name, "bar")
self.assertEqual(service.snapshot, "baz")
self.assertEqual(service.credential, None)
self.assertEqual(service.primary_hostname, 'local-machine:11002/custom/account/path')
self.assertTrue(service.url.startswith('http://local-machine:11002/custom/account/path/foo/bar?snapshot=baz&'))
service = BlobClient.from_blob_url("http://local-machine:11002/custom/account/path/foo/bar?snapshot=baz&query=value")
self.assertEqual(service.account_name, None)
self.assertEqual(service.container_name, "foo")
self.assertEqual(service.blob_name, "bar")
self.assertEqual(service.snapshot, "baz")
self.assertEqual(service.credential, None)
self.assertEqual(service.primary_hostname, 'local-machine:11002/custom/account/path')
self.assertEqual(service.url, 'http://local-machine:11002/custom/account/path/foo/bar?snapshot=baz')
def write_blob(self, container_sas_uri, blob_name, input_stream):
container_client = ContainerClient.from_container_url(
container_sas_uri)
blob_client = container_client.get_blob_client(blob_name)
blob_client.upload_blob(input_stream, overwrite=True)
account_name = self.get_account_from_uri(container_sas_uri)
container_name = self.get_container_from_uri(container_sas_uri)
sas_key = self.get_sas_key_from_uri(container_sas_uri)
return 'https://{}.blob.core.windows.net/{}/{}?{}'.format(
account_name, container_name, blob_name, sas_key)
def upload_image(f):
content_settings = ContentSettings(content_type=f.content_type)
container_client = ContainerClient.from_container_url(
AZURE_CONTAINER_URL_FRONT, AZURE_ACCESS_KEY)
date = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S")
name = f"{date}_{f.name}"
container_client.upload_blob(name,
f,
content_settings=content_settings,
overwrite=False)
return name
def copy_blobs():
ms_blob_url = ""
target_client = ContainerClient.from_container_url(ms_blob_url)
nms_blob_url = ""
source_client = ContainerClient.from_container_url(nms_blob_url)
local_path = "./data"
# Create a local directory to hold blob data
Path(local_path).mkdir(parents=True, exist_ok=True)
# Download all the blobs
for blob in source_client.list_blobs():
download_file_path = os.path.join(local_path, slugify(blob.name))
print("Downloading blob to \n\t" + download_file_path)
with open(download_file_path, "wb") as download_file:
download_file.write(source_client.download_blob(blob).readall())
print("Uploading blob...")
with open(download_file_path, "rb") as data:
target_client.upload_blob(blob, data)
def list_blobs_in(container_url: str, subfolder: str):
container = ContainerClient.from_container_url(container_url=container_url)
print("List files from %s" % container_url)
blob_list = []
try:
for blob in container.list_blobs(name_starts_with=subfolder):
blob_list.append(blob.name)
except AzureMissingResourceHttpError as ce:
print("No blobs found at %s/%s" % (container_url, subfolder))
finally:
container.close()
return blob_list
def create_container_client(account_url, container_name, sas):
container_client = None
try:
container_url = account_url + '/' + container_name + sas
container_client = ContainerClient.from_container_url(container_url)
logging.info(
f"Created container client for container {container_name} in account {account_url}."
)
except Exception as e:
logging.error(
f"Could not create container client for container {container_name} in account {account_url}: {e}"
)
return container_client
def __init__(self, base_path, file_ext, directory_layout='tms', _concurrent_writer = 8, _concurrent_reader = 10, sas_token=None):
super(AzureBlobCache, self).__init__()
self.lock_cache_id = hashlib.md5(base_path.encode('utf-8')).hexdigest()
self.conn = ContainerClient.from_container_url(sas_token)
self.base_path = base_path
self.file_ext = file_ext
self._concurrent_writer = _concurrent_writer
self._concurrent_reader = _concurrent_reader
self._tile_location, _ = path.location_funcs(layout=directory_layout)
def download_blobs(local_path):
sas_url = 'INSERT_URL_HERE'
client = ContainerClient.from_container_url(sas_url)
blob_list = client.list_blobs()
# Create a local directory to hold blob data
Path(local_path).mkdir(parents=True, exist_ok=True)
# Download all the blobs
for blob in blob_list:
download_file_path = os.path.join(local_path, slugify(blob.name))
print("Downloading blob to \n\t" + download_file_path)
with open(download_file_path, "wb") as download_file:
download_file.write(client.download_blob(blob).readall())
def __init__(self, **kwargs):
print('score.py BatchScorer, __init__()')
detector_path = kwargs.get('detector_path')
self.detector = TFDetector(detector_path)
self.use_url = kwargs.get('use_url')
if not self.use_url:
input_container_sas = kwargs.get('input_container_sas')
self.input_container_client = ContainerClient.from_container_url(input_container_sas)
self.detection_threshold = kwargs.get('detection_threshold')
self.image_ids_to_score = kwargs.get('image_ids_to_score')
# determine if there is metadata attached to each image_id
self.metadata_available = True if isinstance(self.image_ids_to_score[0], list) else False
def __init__(self, connection_string: str, container_name: str):
"""Initialize object.
Args:
connection_string (str): Connection string for the storage container
container_name (str): Name of the container to connect to
"""
self.container_name = container_name
try:
self.storage_client = BlobServiceClient.from_connection_string(
connection_string)
self.container_client = self.storage_client.get_container_client(
container_name)
# self.storage_client.get_service_stats()
except:
self.storage_client = None
self.container_client = ContainerClient.from_container_url(
connection_string + "/" + container_name)
print("Read Access only")
def main(req: func.HttpRequest) -> func.HttpResponse:
logging.info('Python HTTP trigger function processed a request.')
local_model_name = "classification_model.h5"
# Create the BlobServiceClient object which will be used to create a container client
blob_service_client = BlobServiceClient.from_connection_string(connect_str)
container_name = "classification-model"
container_client = ContainerClient.from_container_url('https://sigstorageaccount.blob.core.windows.net/classification-model')
logging.info('Created blob service and container client')
blobs = container_client.list_blobs()
for blob in blobs:
logging.info(f'Blob Name: {blob.name}')
with h5py.File(blob.content) as h5_file:
model = tf.keras.models.load_model(h5_file)
logging.info(f'MMMModel: {model}')
# blob_downloader = container_client.download_blob(blob)
# blob_file = blob_downloader.readall()
# logging.info(f'Blob File as Bytes: {blob_file}')
logging.info('Created blob client')
path = req.params.get('path')
if not path:
try:
req_body = req.get_json()
except ValueError:
pass
else:
path = req_body.get('path')
if path:
return func.HttpResponse(f"Hello, {path}. This HTTP triggered function executed successfully.")
else:
return func.HttpResponse(
"This HTTP triggered function executed successfully. Pass a name in the query string or in the request body for a personalized response.",
status_code=200
)
def downloadAll():
try:
blob_client = ContainerClient.from_container_url(container_url) #BlobClient.from_connection_string(conn_str=AZURE_CONNECTION_STRING, container_name='images', blob_name='*')
blobList = blob_client.list_blobs(None, None)
#print(len(blobList))
# Quick start code goes here
for blob in blobList:
#print(blob.name)
# Download the blob to a local file
# Add 'DOWNLOAD' before the .txt extension so you can see both files in Documents
download_file_path = os.path.join(local_path, blob.name )
#print("\nDownloading blob to \n\t" + download_file_path)
#if os.path.exists(download_file_path) != True :
print("\nDownloading blob to \n\t" + download_file_path)
with open(download_file_path, "wb") as download_file:
download_file.write(blob_client.download_blob(blob).readall())
except Exception as ex:
print('Exception:')
print(ex)
def push_image(picture, name, age, category):
container = ContainerClient.from_container_url(sas)
item_id = uuid.uuid1()
blob_name = str(item_id) + '.' + picture.name.split('.')[1]
pil_im = Image.open(picture)
b = io.BytesIO()
pil_im.save(b, 'jpeg')
im_bytes = b.getvalue()
container.upload_blob(name=blob_name, data=im_bytes)
item_create = {
'id':
str(item_id),
'name':
name,
'age':
age,
'category':
category,
'picture':
'https://lab5storagebekbolat.blob.core.windows.net/images/' + blob_name
}
return item_create
async def iot_download_model(request, device_client, credentials):
global log
log.info("IoT Download Model called")
response = MethodResponse.create_from_method_request(request, status=200)
asyncio.sleep(0)
await device_client.send_method_response(response) # send response
# Setup blob service and container client for downloading from the blob storage
container = ContainerClient.from_container_url(credentials.tf_models)
blob_list = container.list_blobs()
for blob in blob_list:
if blob.name.endswith(".tflite") or blob.name == "signature.json":
download_path = os.path.join('./tf_models_lite', blob.name)
log.info("Downloading %s to %s" % (blob.name, download_path))
blob_client = container.get_blob_client(blob)
with open(download_path, "wb") as local_file:
download_stream = blob_client.download_blob()
local_file.write(download_stream.readall())
else:
pass
async def iot_upload_images(request, device_client, credentials):
global log
log.info("IoT Upload Images called")
response = MethodResponse.create_from_method_request(request, status=200)
asyncio.sleep(0)
await device_client.send_method_response(response) # send response
# Setup blob service and container client for downloading from the blob storage
container = ContainerClient.from_container_url(credentials.blob_token)
for filename in os.listdir('img'):
filepath = 'img/' + filename
if filename.endswith('.jpg'):
blob = container.get_blob_client(filename)
# Upload content to block blob
with open(filepath, "rb") as data:
log.info("Uploading %s to blob storage" % (filename))
blob.upload_blob(data, blob_type="BlockBlob")
# Delete the file once it was uploaded
if os.path.exists(filepath):
os.remove(filepath)
def __init__(self):
# Configuration option 2:
if BlobFileSystem.SAS_URI:
self.connection_string = 'BlobEndpoint={};'.format(
BlobFileSystem.SAS_URI)
self.container_name = urlparse(BlobFileSystem.SAS_URI).path
self.container_name = self.container_name[
self.container_name.rindex('/') +
1:] # strip everything up to the last '/'
self.credentials = None
self.blob_service_client = None
self.container_client = ContainerClient.from_container_url(
BlobFileSystem.SAS_URI)
# Configuration option 1:
else:
self.connection_string = BlobFileSystem.CONNECTION_STRING
self.container_name = BlobFileSystem.CONTAINER_NAME
self.credentials = BlobFileSystem.CREDENTIALS
self.blob_service_client = BlobServiceClient.from_connection_string(
self.connection_string, credential=self.credentials)
self.container_client = self._get_container_client(
self.container_name, ensure_creation=True)
def tearDown(self):
if self.needs_cleanup:
# cleanup: delete the private emulated container
print('running cleanup')
with BlobClient(account_url=PRIVATE_ACCOUNT_URI,
container_name=PRIVATE_CONTAINER_NAME,
blob_name=PRIVATE_BLOB_NAME,
credential=PRIVATE_ACCOUNT_KEY) as bc:
if bc.exists():
print('deleted blob')
bc.delete_blob(delete_snapshots='include')
with ContainerClient.from_container_url(
PRIVATE_CONTAINER_URI,
credential=PRIVATE_ACCOUNT_KEY) as cc:
try:
cc.get_container_properties()
cc.delete_container()
print('deleted container')
except ResourceNotFoundError:
pass
self.needs_cleanup = False
def get_container_client(self, hex_obj_id):
"""Get the block_blob_service and container that contains the object with
internal id hex_obj_id
"""
prefix = hex_obj_id[: self.prefix_len]
return ContainerClient.from_container_url(self.container_urls[prefix])
######## Author : Manoj Babu G R #########
#####################################################################################################
from azure.storage.blob import BlobServiceClient, ContainerClient, BlobClient
import pandas as pd
import json
from datetime import datetime
from calendar import timegm
# Credentials-----------------------------------------------------------------------------------------------------------
# setting up connection to container which has the recieving blobs
service = BlobServiceClient(account_url="xxxxx", credential="xxx")
sas_url = "xxx"
container = ContainerClient.from_container_url(sas_url)
container_name = 'input'
# setting up blob client to retrieve properties of each blob and append to dictionary
conn_str = 'xxx'
# All the Variables
list_of_all_blobs = []
list_of_blobs_Csv = []
old_file_names = []
final_list_of_files = []
delta_load_file_names = {}
# function convert datetime.datetime utc to unixlike timestamp
def timestamping(string):
