async def archive(self, payload: Payload, request: Request) -> ArchiverResponse: """ Archive payload to Azure Blob Storage """ if self.use_sha: filename = hashlib.sha1(payload.content).hexdigest() filename = f'{"/".join(list(filename[:5]))}/{filename}' elif self.use_datetime: datetime_path = datetime.now().strftime('%Y/%m/%d') filename = f'{datetime_path}/{payload.payload_id}' else: filename = payload.results.payload_id blob_client: BlobClient = BlobClient.from_connection_string( conn_str=self.conn_str, container_name=self.archive_container, blob_name=filename, ) try: await blob_client.upload_blob(payload.content) except ResourceExistsError: pass await blob_client.close() return ArchiverResponse({ 'container_name': self.archive_container, 'blob_name': filename })
async def main(event: func.EventGridEvent): result = json.dumps({ 'id': event.id, 'data': event.get_json(), 'topic': event.topic, 'subject': event.subject, 'event_type': event.event_type, }) logging.info('Python EventGrid trigger processed an event: %s', result) try: result = json.loads(result) data = result['data'] url = urlparse(data['url']) p_file = Path(url.path) filename = p_file.name container = p_file.parent.name connection = os.environ['STORAGE_CONNECTION'] async with BlobClient.from_connection_string( connection, container_name=container, blob_name=filename) as blob: stream = await blob.download_blob() data = await stream.content_as_text() logging.info(f'Content is {data}') except Exception as e: logging.exception(f"failed to load EventGrid request:{e}")
def __init__(self): id = uuid.uuid1() connectionString = os.environ["STORAGE_CONNECTION_STRING"] self.blob = BlobClient.from_connection_string( conn_str=connectionString, container_name="mycontainer", blob_name="pyTestBlob-" + id.hex + ".txt", )
async def save(self, response: StoqResponse) -> None: """ Save response as Azure Blob Storage """ blob_client: BlobClient = BlobClient.from_connection_string( conn_str=self.conn_str, container_name=self.results_container, blob_name=response.scan_id, ) await blob_client.upload_blob(dumps(response)) await blob_client.close()
async def upload_file(container, filename, cs): print('Uploading Blob Start') blob = BlobClient.from_connection_string( conn_str=cs, container_name=container, blob_name=os.path.basename(filename)) with open(filename, "rb") as data: await blob.upload_blob(data) print('Uploading Blob End') os.remove(filename)
async def get(self, task: ArchiverResponse) -> Payload: """ Retrieve archived payload from Azure Blob Storage """ blob_client: BlobClient = BlobClient.from_connection_string( conn_str=self.conn_str, container_name=task.results['container_name'], blob_name=task.results['blob_name'], ) content = await blob_client.download_blob() await blob_client.close() meta = PayloadMeta(task.results) return Payload(content.readall(), meta)
def __init__(self, container: str, path: str = str(), connection_string: str = STORAGE_CONNECTION_STRING, content_type: Union[str, None] = DEFAULT_CONTENT_TYPE, cache_control: str = DEFAULT_CACHE_CONTROL, compressed: bool = True, content_disposition: Union[str, None] = None, content_language: Union[str, None] = CONTENT_LANGUAGE, tier: str = 'Hot', **kwargs): self.path = path self.compressed = compressed self._connection_string = connection_string self.container = container self._tier = getattr(StandardBlobTier, tier, None) self._lock = None if self._tier is None: raise ValueError( "Tier must be one of 'Hot', 'Cool' or 'Archive'. " "Got <%r> instead." % tier ) self._content_settings: ContentSettings = ContentSettings( content_type=content_type, cache_control=cache_control, content_encoding="gzip" if self.compressed else None, content_language=content_language, content_disposition=content_disposition, **kwargs ) self.client: AsyncBlobClient = AsyncBlobClient.from_connection_string( conn_str=connection_string, container_name=container, blob_name=path, # retry_to_secondary=True, connection_timeout=60, max_block_size=8 * 1024 * 1024, max_single_put_size=256 * 1024 * 1024, min_large_block_upload_threshold=8 * 1024 * 1024 + 1 ) # self.client.blob_name self.account_name = self.client.account_name self.target = self.client.primary_hostname self.url = "{}://{}/{}/{}".format( self.client.scheme, self.client.primary_hostname, quote(self.container), quote(self.path, safe='~/'), )
def upload_blob_file(connection,container_name,local_file_name, full_path): """ Upload local file to cloud. :param Object connection: Azure connection instance :param str container_name: Azure Container name. :param str local_file_name: File name to upload. :param str full_path: Path file that will be uploaded. :return: True if upload works and False if something went wrong. :rtype: Boolean """ try: blob = BlobClient.from_connection_string(conn_str=connection, container_name=container_name, blob_name=local_file_name) with open(full_path,'rb') as data: blob.upload_blob(data) print('File {0} uploaded'.format(local_file_name)) return True except ValueError: print("Something went wrong") return False
async def _test_auth_connection_string_async(self): # [START auth_from_connection_string] from azure.storage.blob.aio import BlobServiceClient blob_service_client = BlobServiceClient.from_connection_string(self.connection_string) # [END auth_from_connection_string] # [START auth_from_connection_string_container] from azure.storage.blob.aio import ContainerClient container_client = ContainerClient.from_connection_string( self.connection_string, container_name="mycontainer") # [END auth_from_connection_string_container] # [START auth_from_connection_string_blob] from azure.storage.blob.aio import BlobClient blob_client = BlobClient.from_connection_string( self.connection_string, container_name="mycontainer", blob_name="blobname.txt") # [END auth_from_connection_string_blob] # Get account information for the Blob Service account_info = await blob_service_client.get_account_information() assert account_info is not None
def download_blob_file(connection,container_name,blob_name,path_output): """ Get blob file from cloud and downloads on local machine. :param Object connection: Azure connection instance :param str container_name: Azure Container name. :param str blob_name: Azure Blob name to download. :param str blob_name: Path where file will be saved. :return: True if download works and False if something went wrong. :rtype: Boolean """ try: blob = BlobClient.from_connection_string(conn_str=connection, container_name=container_name, blob_name=blob_name) with open(path_output,'wb') as myblob: blob_data = blob.download_blob() blob_data.readinto(myblob) print('File saved in {0}'.format(path_output)) return True except ValueError: print("Something went wrong") return False