def main(req: func.HttpRequest) -> func.HttpResponse: logging.info(req.get_json()) try: is_pepper_include = req.get_json()['pepper'] except: return func.HttpResponse("Write down whether to include pepper or not", status_code=400) try: if is_pepper_include: url = os.environ["PEPPER_CONTAINER"] container = ContainerClient.from_container_url(url) else: url = os.environ["NO_PEPPER_CONTAINER"] container = ContainerClient.from_container_url(url) blob_list = list(container.list_blobs()) blob = choice(blob_list) except Exception as e: logging.info("Err: {}".format(e)) return func.HttpResponse(str(e), status_code=500) image_url = blob.name return_data = {"tteokgukImageUrl": url + image_url} return func.HttpResponse(json.dumps(return_data), headers={"Content-Type": "application/json"}, status_code=200)
def test_create_client_for_emulator(self): container_client = ContainerClient( account_url='http://127.0.0.1:1000/devstoreaccount1', container_name='newcontainer', credential= 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==' ) self.assertEqual(container_client.container_name, "newcontainer") self.assertEqual(container_client.account_name, "devstoreaccount1") ContainerClient.from_container_url( 'http://127.0.0.1:1000/devstoreaccount1/newcontainer') self.assertEqual(container_client.container_name, "newcontainer") self.assertEqual(container_client.account_name, "devstoreaccount1")
def test_sas_signature_is_scrubbed_off(self, resource_group, location, storage_account, storage_account_key): # SAS URL is calculated from storage key, so this test runs live only bsc = BlobServiceClient(self.account_url(storage_account, "blob"), storage_account_key) self._setup(bsc) # Arrange container = bsc.get_container_client(self.container_name) token = generate_container_sas( container.account_name, container.container_name, account_key=container.credential.account_key, permission=ContainerSasPermissions(read=True), expiry=datetime.utcnow() + timedelta(hours=1), ) # parse out the signed signature token_components = parse_qs(token) signed_signature = quote( token_components[QueryStringConstants.SIGNED_SIGNATURE][0]) sas_service = ContainerClient.from_container_url(container.url, credential=token) # Act with LogCaptured(self) as log_captured: sas_service.get_account_information(logging_enable=True) log_as_str = log_captured.getvalue() # Assert # make sure the query parameter 'sig' is logged, but its value is not self.assertTrue( QueryStringConstants.SIGNED_SIGNATURE in log_as_str) self.assertFalse(signed_signature in log_as_str)
def container_access_policy(self): # SAS URL is calculated from storage key, so this test runs live only if TestMode.need_recording_file(self.test_mode): return # Instantiate a BlobServiceClient using a connection string from azure.storage.blob import BlobServiceClient blob_service_client = BlobServiceClient.from_connection_string( self.connection_string) # Instantiate a ContainerClient container_client = blob_service_client.get_container_client( "myaccesscontainer") try: # Create new Container container_client.create_container() # [START set_container_access_policy] # Create access policy from azure.storage.blob import AccessPolicy, ContainerSasPermissions access_policy = AccessPolicy( permission=ContainerSasPermissions(read=True), expiry=datetime.utcnow() + timedelta(hours=1), start=datetime.utcnow() - timedelta(minutes=1)) identifiers = {'test': access_policy} # Set the access policy on the container container_client.set_container_access_policy( signed_identifiers=identifiers) # [END set_container_access_policy] # [START get_container_access_policy] policy = container_client.get_container_access_policy() # [END get_container_access_policy] # [START generate_sas_token] # Use access policy to generate a sas token from azure.storage.blob import generate_container_sas sas_token = generate_container_sas( container_client.account_name, container_client.container_name, account_key=container_client.credential.account_key, policy_id='my-access-policy-id') # [END generate_sas_token] # Use the sas token to authenticate a new client # [START create_container_client_sastoken] from azure.storage.blob import ContainerClient container = ContainerClient.from_container_url( container_url= "https://account.blob.core.windows.net/mycontainer", credential=sas_token) # [END create_container_client_sastoken] finally: # Delete container container_client.delete_container()
def test_sas_signature_is_scrubbed_off(self): # SAS URL is calculated from storage key, so this test runs live only if TestMode.need_recording_file(self.test_mode): return # Arrange container = self.bsc.get_container_client(self.container_name) token = container.generate_shared_access_signature( permission=ContainerSasPermissions(read=True), expiry=datetime.utcnow() + timedelta(hours=1), ) # parse out the signed signature token_components = parse_qs(token) signed_signature = quote( token_components[QueryStringConstants.SIGNED_SIGNATURE][0]) sas_service = ContainerClient.from_container_url(container.url, credential=token) # Act with LogCaptured(self) as log_captured: sas_service.get_account_information(logging_enable=True) log_as_str = log_captured.getvalue() # Assert # make sure the query parameter 'sig' is logged, but its value is not self.assertTrue( QueryStringConstants.SIGNED_SIGNATURE in log_as_str) self.assertFalse(signed_signature in log_as_str)
def enumerate_files(dir: str, ext: str, credentials: Optional[Union[str,Dict[str,str]]]): blob_data = _try_parse_azure_blob_uri(dir) if blob_data is None: return [os.path.join(dir, path.name) for path in os.scandir(dir) if path.is_file() and (ext is None or path.name.endswith(ext))] else: try: # pip install azure-storage-blob from azure.storage.blob import ContainerClient except: print("Failed to import azure.storage.blob. Please pip install azure-storage-blob", file=sys.stderr) raise account, container, blob_path = blob_data print("enumerate_files: enumerating blobs in", dir, file=sys.stderr, flush=True) # @BUGBUG: The prefix does not seem to have to start; seems it can also be a substring container_uri = "https://" + account + ".blob.core.windows.net/" + container container_client = ContainerClient.from_container_url(container_uri, credential=_get_azure_key(account, credentials)) if not blob_path.endswith("/"): blob_path += "/" blob_uris = [container_uri + "/" + blob["name"] for blob in container_client.walk_blobs(blob_path, delimiter="") if (ext is None or blob["name"].endswith(ext))] print("enumerate_files:", len(blob_uris), "blobs found", file=sys.stderr, flush=True) for blob_name in blob_uris[:10]: print(blob_name, file=sys.stderr, flush=True) return blob_uris
def CopyImages(): """Ingest the images to the data lake. """ print('Transferring images to Data Lake.') container = ContainerClient.from_container_url( container_url="https://bus5wb.blob.core.windows.net/imagecollection", credential= "?st=2021-05-19T04%3A43%3A08Z&se=2022-05-20T04%3A43%3A00Z&sp=rl&sv=2018-03-28&sr=c&sig=yV19KD0EzGYQOecRpa2em6Fc6IRQ7%2FhowiAaO%2Bk70O4%3D" ) container_client = ContainerClient.from_connection_string( connection_string, container_name=container_name_images) blobs_list = container.list_blobs() for blob in blobs_list: blob_client = container.get_blob_client(blob.name) nycImageUrl = blob_client.url # Generate filename for image metadata file. filePath = urlparse(nycImageUrl)[2] filePathParts = filePath.split('/') fileName = filePathParts[len(filePathParts) - 1] try: # Instantiate a new BlobClient blob_client = container_client.get_blob_client(fileName) # Upload the blob blob_client.start_copy_from_url(nycImageUrl) except Exception as e: print(f"Error {e} - {fileName} - {nycImageUrl}")
def container_sample(self): # [START create_container_client_from_service] # Instantiate a BlobServiceClient using a connection string from azure.storage.blob import BlobServiceClient blob_service_client = BlobServiceClient.from_connection_string( self.connection_string) # Instantiate a ContainerClient container_client = blob_service_client.get_container_client( "mynewcontainer") # [END create_container_client_from_service] # [START create_container_client_sasurl] from azure.storage.blob import ContainerClient sas_url = "https://account.blob.core.windows.net/mycontainer?sv=2015-04-05&st=2015-04-29T22%3A18%3A26Z&se=2015-04-30T02%3A23%3A26Z&sr=b&sp=rw&sip=168.1.5.60-168.1.5.70&spr=https&sig=Z%2FRHIX5Xcg0Mq2rqI3OlWTjEg2tYkboXr1P9ZUXDtkk%3D" container = ContainerClient.from_container_url(sas_url) # [END create_container_client_sasurl] try: # [START create_container] container_client.create_container() # [END create_container] # [START get_container_properties] properties = container_client.get_container_properties() # [END get_container_properties] finally: # [START delete_container] container_client.delete_container()
def create_container_client(self, account, container): if self.connection_string: client = ContainerClient.from_connection_string( self.connection_string, container) else: client = ContainerClient.from_container_url("https://{}/{}".format( account, container)) return client
def get_container_client(self, hex_obj_id): """Get the container client for the container that contains the object with internal id hex_obj_id This is used to allow the PrefixedAzureCloudObjStorage to dispatch the client according to the prefix of the object id. """ return ContainerClient.from_container_url(self.container_url)
def get_blob_sas_uri(self, container_sas_uri, blob_name): container_client = ContainerClient.from_container_url( container_sas_uri) blob_client = container_client.get_blob_client(blob_name) account_name = self.get_account_from_uri(container_sas_uri) container_name = self.get_container_from_uri(container_sas_uri) sas_key = self.get_sas_key_from_uri(container_sas_uri) return 'https://{}.blob.core.windows.net/{}/{}?{}'.format( account_name, container_name, blob_name, sas_key)
def test_create_service_with_custom_account_endpoint_path(self): account_name = "blobstorage" account_key = "blobkey" custom_account_url = "http://local-machine:11002/custom/account/path/" + self.sas_token for service_type in SERVICES.items(): conn_string = 'DefaultEndpointsProtocol=http;AccountName={};AccountKey={};BlobEndpoint={};'.format( account_name, account_key, custom_account_url) # Act service = service_type[0].from_connection_string( conn_string, container_name="foo", blob_name="bar") # Assert self.assertEqual(service.account_name, account_name) self.assertEqual(service.credential.account_name, account_name) self.assertEqual(service.credential.account_key, account_key) self.assertEqual(service.primary_hostname, 'local-machine:11002/custom/account/path') service = BlobServiceClient(account_url=custom_account_url) self.assertEqual(service.account_name, None) self.assertEqual(service.credential, None) self.assertEqual(service.primary_hostname, 'local-machine:11002/custom/account/path') self.assertTrue(service.url.startswith('http://local-machine:11002/custom/account/path/?')) service = ContainerClient(account_url=custom_account_url, container_name="foo") self.assertEqual(service.account_name, None) self.assertEqual(service.container_name, "foo") self.assertEqual(service.credential, None) self.assertEqual(service.primary_hostname, 'local-machine:11002/custom/account/path') self.assertTrue(service.url.startswith('http://local-machine:11002/custom/account/path/foo?')) service = ContainerClient.from_container_url("http://local-machine:11002/custom/account/path/foo?query=value") self.assertEqual(service.account_name, None) self.assertEqual(service.container_name, "foo") self.assertEqual(service.credential, None) self.assertEqual(service.primary_hostname, 'local-machine:11002/custom/account/path') self.assertEqual(service.url, 'http://local-machine:11002/custom/account/path/foo') service = BlobClient(account_url=custom_account_url, container_name="foo", blob_name="bar", snapshot="baz") self.assertEqual(service.account_name, None) self.assertEqual(service.container_name, "foo") self.assertEqual(service.blob_name, "bar") self.assertEqual(service.snapshot, "baz") self.assertEqual(service.credential, None) self.assertEqual(service.primary_hostname, 'local-machine:11002/custom/account/path') self.assertTrue(service.url.startswith('http://local-machine:11002/custom/account/path/foo/bar?snapshot=baz&')) service = BlobClient.from_blob_url("http://local-machine:11002/custom/account/path/foo/bar?snapshot=baz&query=value") self.assertEqual(service.account_name, None) self.assertEqual(service.container_name, "foo") self.assertEqual(service.blob_name, "bar") self.assertEqual(service.snapshot, "baz") self.assertEqual(service.credential, None) self.assertEqual(service.primary_hostname, 'local-machine:11002/custom/account/path') self.assertEqual(service.url, 'http://local-machine:11002/custom/account/path/foo/bar?snapshot=baz')
def write_blob(self, container_sas_uri, blob_name, input_stream): container_client = ContainerClient.from_container_url( container_sas_uri) blob_client = container_client.get_blob_client(blob_name) blob_client.upload_blob(input_stream, overwrite=True) account_name = self.get_account_from_uri(container_sas_uri) container_name = self.get_container_from_uri(container_sas_uri) sas_key = self.get_sas_key_from_uri(container_sas_uri) return 'https://{}.blob.core.windows.net/{}/{}?{}'.format( account_name, container_name, blob_name, sas_key)
def upload_image(f): content_settings = ContentSettings(content_type=f.content_type) container_client = ContainerClient.from_container_url( AZURE_CONTAINER_URL_FRONT, AZURE_ACCESS_KEY) date = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S") name = f"{date}_{f.name}" container_client.upload_blob(name, f, content_settings=content_settings, overwrite=False) return name
def copy_blobs(): ms_blob_url = "" target_client = ContainerClient.from_container_url(ms_blob_url) nms_blob_url = "" source_client = ContainerClient.from_container_url(nms_blob_url) local_path = "./data" # Create a local directory to hold blob data Path(local_path).mkdir(parents=True, exist_ok=True) # Download all the blobs for blob in source_client.list_blobs(): download_file_path = os.path.join(local_path, slugify(blob.name)) print("Downloading blob to \n\t" + download_file_path) with open(download_file_path, "wb") as download_file: download_file.write(source_client.download_blob(blob).readall()) print("Uploading blob...") with open(download_file_path, "rb") as data: target_client.upload_blob(blob, data)
def list_blobs_in(container_url: str, subfolder: str): container = ContainerClient.from_container_url(container_url=container_url) print("List files from %s" % container_url) blob_list = [] try: for blob in container.list_blobs(name_starts_with=subfolder): blob_list.append(blob.name) except AzureMissingResourceHttpError as ce: print("No blobs found at %s/%s" % (container_url, subfolder)) finally: container.close() return blob_list
def create_container_client(account_url, container_name, sas): container_client = None try: container_url = account_url + '/' + container_name + sas container_client = ContainerClient.from_container_url(container_url) logging.info( f"Created container client for container {container_name} in account {account_url}." ) except Exception as e: logging.error( f"Could not create container client for container {container_name} in account {account_url}: {e}" ) return container_client
def __init__(self, base_path, file_ext, directory_layout='tms', _concurrent_writer = 8, _concurrent_reader = 10, sas_token=None): super(AzureBlobCache, self).__init__() self.lock_cache_id = hashlib.md5(base_path.encode('utf-8')).hexdigest() self.conn = ContainerClient.from_container_url(sas_token) self.base_path = base_path self.file_ext = file_ext self._concurrent_writer = _concurrent_writer self._concurrent_reader = _concurrent_reader self._tile_location, _ = path.location_funcs(layout=directory_layout)
def download_blobs(local_path): sas_url = 'INSERT_URL_HERE' client = ContainerClient.from_container_url(sas_url) blob_list = client.list_blobs() # Create a local directory to hold blob data Path(local_path).mkdir(parents=True, exist_ok=True) # Download all the blobs for blob in blob_list: download_file_path = os.path.join(local_path, slugify(blob.name)) print("Downloading blob to \n\t" + download_file_path) with open(download_file_path, "wb") as download_file: download_file.write(client.download_blob(blob).readall())
def __init__(self, **kwargs): print('score.py BatchScorer, __init__()') detector_path = kwargs.get('detector_path') self.detector = TFDetector(detector_path) self.use_url = kwargs.get('use_url') if not self.use_url: input_container_sas = kwargs.get('input_container_sas') self.input_container_client = ContainerClient.from_container_url(input_container_sas) self.detection_threshold = kwargs.get('detection_threshold') self.image_ids_to_score = kwargs.get('image_ids_to_score') # determine if there is metadata attached to each image_id self.metadata_available = True if isinstance(self.image_ids_to_score[0], list) else False
def __init__(self, connection_string: str, container_name: str): """Initialize object. Args: connection_string (str): Connection string for the storage container container_name (str): Name of the container to connect to """ self.container_name = container_name try: self.storage_client = BlobServiceClient.from_connection_string( connection_string) self.container_client = self.storage_client.get_container_client( container_name) # self.storage_client.get_service_stats() except: self.storage_client = None self.container_client = ContainerClient.from_container_url( connection_string + "/" + container_name) print("Read Access only")
def main(req: func.HttpRequest) -> func.HttpResponse: logging.info('Python HTTP trigger function processed a request.') local_model_name = "classification_model.h5" # Create the BlobServiceClient object which will be used to create a container client blob_service_client = BlobServiceClient.from_connection_string(connect_str) container_name = "classification-model" container_client = ContainerClient.from_container_url('https://sigstorageaccount.blob.core.windows.net/classification-model') logging.info('Created blob service and container client') blobs = container_client.list_blobs() for blob in blobs: logging.info(f'Blob Name: {blob.name}') with h5py.File(blob.content) as h5_file: model = tf.keras.models.load_model(h5_file) logging.info(f'MMMModel: {model}') # blob_downloader = container_client.download_blob(blob) # blob_file = blob_downloader.readall() # logging.info(f'Blob File as Bytes: {blob_file}') logging.info('Created blob client') path = req.params.get('path') if not path: try: req_body = req.get_json() except ValueError: pass else: path = req_body.get('path') if path: return func.HttpResponse(f"Hello, {path}. This HTTP triggered function executed successfully.") else: return func.HttpResponse( "This HTTP triggered function executed successfully. Pass a name in the query string or in the request body for a personalized response.", status_code=200 )
def downloadAll(): try: blob_client = ContainerClient.from_container_url(container_url) #BlobClient.from_connection_string(conn_str=AZURE_CONNECTION_STRING, container_name='images', blob_name='*') blobList = blob_client.list_blobs(None, None) #print(len(blobList)) # Quick start code goes here for blob in blobList: #print(blob.name) # Download the blob to a local file # Add 'DOWNLOAD' before the .txt extension so you can see both files in Documents download_file_path = os.path.join(local_path, blob.name ) #print("\nDownloading blob to \n\t" + download_file_path) #if os.path.exists(download_file_path) != True : print("\nDownloading blob to \n\t" + download_file_path) with open(download_file_path, "wb") as download_file: download_file.write(blob_client.download_blob(blob).readall()) except Exception as ex: print('Exception:') print(ex)
def push_image(picture, name, age, category): container = ContainerClient.from_container_url(sas) item_id = uuid.uuid1() blob_name = str(item_id) + '.' + picture.name.split('.')[1] pil_im = Image.open(picture) b = io.BytesIO() pil_im.save(b, 'jpeg') im_bytes = b.getvalue() container.upload_blob(name=blob_name, data=im_bytes) item_create = { 'id': str(item_id), 'name': name, 'age': age, 'category': category, 'picture': 'https://lab5storagebekbolat.blob.core.windows.net/images/' + blob_name } return item_create
async def iot_download_model(request, device_client, credentials): global log log.info("IoT Download Model called") response = MethodResponse.create_from_method_request(request, status=200) asyncio.sleep(0) await device_client.send_method_response(response) # send response # Setup blob service and container client for downloading from the blob storage container = ContainerClient.from_container_url(credentials.tf_models) blob_list = container.list_blobs() for blob in blob_list: if blob.name.endswith(".tflite") or blob.name == "signature.json": download_path = os.path.join('./tf_models_lite', blob.name) log.info("Downloading %s to %s" % (blob.name, download_path)) blob_client = container.get_blob_client(blob) with open(download_path, "wb") as local_file: download_stream = blob_client.download_blob() local_file.write(download_stream.readall()) else: pass
async def iot_upload_images(request, device_client, credentials): global log log.info("IoT Upload Images called") response = MethodResponse.create_from_method_request(request, status=200) asyncio.sleep(0) await device_client.send_method_response(response) # send response # Setup blob service and container client for downloading from the blob storage container = ContainerClient.from_container_url(credentials.blob_token) for filename in os.listdir('img'): filepath = 'img/' + filename if filename.endswith('.jpg'): blob = container.get_blob_client(filename) # Upload content to block blob with open(filepath, "rb") as data: log.info("Uploading %s to blob storage" % (filename)) blob.upload_blob(data, blob_type="BlockBlob") # Delete the file once it was uploaded if os.path.exists(filepath): os.remove(filepath)
def __init__(self): # Configuration option 2: if BlobFileSystem.SAS_URI: self.connection_string = 'BlobEndpoint={};'.format( BlobFileSystem.SAS_URI) self.container_name = urlparse(BlobFileSystem.SAS_URI).path self.container_name = self.container_name[ self.container_name.rindex('/') + 1:] # strip everything up to the last '/' self.credentials = None self.blob_service_client = None self.container_client = ContainerClient.from_container_url( BlobFileSystem.SAS_URI) # Configuration option 1: else: self.connection_string = BlobFileSystem.CONNECTION_STRING self.container_name = BlobFileSystem.CONTAINER_NAME self.credentials = BlobFileSystem.CREDENTIALS self.blob_service_client = BlobServiceClient.from_connection_string( self.connection_string, credential=self.credentials) self.container_client = self._get_container_client( self.container_name, ensure_creation=True)
def tearDown(self): if self.needs_cleanup: # cleanup: delete the private emulated container print('running cleanup') with BlobClient(account_url=PRIVATE_ACCOUNT_URI, container_name=PRIVATE_CONTAINER_NAME, blob_name=PRIVATE_BLOB_NAME, credential=PRIVATE_ACCOUNT_KEY) as bc: if bc.exists(): print('deleted blob') bc.delete_blob(delete_snapshots='include') with ContainerClient.from_container_url( PRIVATE_CONTAINER_URI, credential=PRIVATE_ACCOUNT_KEY) as cc: try: cc.get_container_properties() cc.delete_container() print('deleted container') except ResourceNotFoundError: pass self.needs_cleanup = False
def get_container_client(self, hex_obj_id): """Get the block_blob_service and container that contains the object with internal id hex_obj_id """ prefix = hex_obj_id[: self.prefix_len] return ContainerClient.from_container_url(self.container_urls[prefix])
######## Author : Manoj Babu G R ######### ##################################################################################################### from azure.storage.blob import BlobServiceClient, ContainerClient, BlobClient import pandas as pd import json from datetime import datetime from calendar import timegm # Credentials----------------------------------------------------------------------------------------------------------- # setting up connection to container which has the recieving blobs service = BlobServiceClient(account_url="xxxxx", credential="xxx") sas_url = "xxx" container = ContainerClient.from_container_url(sas_url) container_name = 'input' # setting up blob client to retrieve properties of each blob and append to dictionary conn_str = 'xxx' # All the Variables list_of_all_blobs = [] list_of_blobs_Csv = [] old_file_names = [] final_list_of_files = [] delta_load_file_names = {} # function convert datetime.datetime utc to unixlike timestamp def timestamping(string):