async def archive(self, payload: Payload,
request: Request) -> ArchiverResponse:
"""
Archive payload to Azure Blob Storage
"""
if self.use_sha:
filename = hashlib.sha1(payload.content).hexdigest()
filename = f'{"/".join(list(filename[:5]))}/{filename}'
elif self.use_datetime:
datetime_path = datetime.now().strftime('%Y/%m/%d')
filename = f'{datetime_path}/{payload.payload_id}'
else:
filename = payload.results.payload_id
blob_client: BlobClient = BlobClient.from_connection_string(
conn_str=self.conn_str,
container_name=self.archive_container,
blob_name=filename,
)
try:
await blob_client.upload_blob(payload.content)
except ResourceExistsError:
pass
await blob_client.close()
return ArchiverResponse({
'container_name': self.archive_container,
'blob_name': filename
})
async def main(event: func.EventGridEvent):
result = json.dumps({
'id': event.id,
'data': event.get_json(),
'topic': event.topic,
'subject': event.subject,
'event_type': event.event_type,
})
logging.info('Python EventGrid trigger processed an event: %s', result)
try:
result = json.loads(result)
data = result['data']
url = urlparse(data['url'])
p_file = Path(url.path)
filename = p_file.name
container = p_file.parent.name
connection = os.environ['STORAGE_CONNECTION']
async with BlobClient.from_connection_string(
connection, container_name=container,
blob_name=filename) as blob:
stream = await blob.download_blob()
data = await stream.content_as_text()
logging.info(f'Content is {data}')
except Exception as e:
logging.exception(f"failed to load EventGrid request:{e}")
def __init__(self):
id = uuid.uuid1()
connectionString = os.environ["STORAGE_CONNECTION_STRING"]
self.blob = BlobClient.from_connection_string(
conn_str=connectionString,
container_name="mycontainer",
blob_name="pyTestBlob-" + id.hex + ".txt",
)
async def save(self, response: StoqResponse) -> None:
"""
Save response as Azure Blob Storage
"""
blob_client: BlobClient = BlobClient.from_connection_string(
conn_str=self.conn_str,
container_name=self.results_container,
blob_name=response.scan_id,
)
await blob_client.upload_blob(dumps(response))
await blob_client.close()
async def upload_file(container, filename, cs):
print('Uploading Blob Start')
blob = BlobClient.from_connection_string(
conn_str=cs,
container_name=container,
blob_name=os.path.basename(filename))
with open(filename, "rb") as data:
await blob.upload_blob(data)
print('Uploading Blob End')
os.remove(filename)
async def get(self, task: ArchiverResponse) -> Payload:
"""
Retrieve archived payload from Azure Blob Storage
"""
blob_client: BlobClient = BlobClient.from_connection_string(
conn_str=self.conn_str,
container_name=task.results['container_name'],
blob_name=task.results['blob_name'],
)
content = await blob_client.download_blob()
await blob_client.close()
meta = PayloadMeta(task.results)
return Payload(content.readall(), meta)
def __init__(self, container: str, path: str = str(),
connection_string: str = STORAGE_CONNECTION_STRING,
content_type: Union[str, None] = DEFAULT_CONTENT_TYPE,
cache_control: str = DEFAULT_CACHE_CONTROL, compressed: bool = True,
content_disposition: Union[str, None] = None,
content_language: Union[str, None] = CONTENT_LANGUAGE,
tier: str = 'Hot', **kwargs):
self.path = path
self.compressed = compressed
self._connection_string = connection_string
self.container = container
self._tier = getattr(StandardBlobTier, tier, None)
self._lock = None
if self._tier is None:
raise ValueError(
"Tier must be one of 'Hot', 'Cool' or 'Archive'. "
"Got <%r> instead." % tier
)
self._content_settings: ContentSettings = ContentSettings(
content_type=content_type,
cache_control=cache_control,
content_encoding="gzip" if self.compressed else None,
content_language=content_language,
content_disposition=content_disposition,
**kwargs
)
self.client: AsyncBlobClient = AsyncBlobClient.from_connection_string(
conn_str=connection_string,
container_name=container,
blob_name=path,
# retry_to_secondary=True,
connection_timeout=60,
max_block_size=8 * 1024 * 1024,
max_single_put_size=256 * 1024 * 1024,
min_large_block_upload_threshold=8 * 1024 * 1024 + 1
)
# self.client.blob_name
self.account_name = self.client.account_name
self.target = self.client.primary_hostname
self.url = "{}://{}/{}/{}".format(
self.client.scheme,
self.client.primary_hostname,
quote(self.container),
quote(self.path, safe='~/'),
)
def upload_blob_file(connection,container_name,local_file_name, full_path):
"""
Upload local file to cloud.
:param Object connection: Azure connection instance
:param str container_name: Azure Container name.
:param str local_file_name: File name to upload.
:param str full_path: Path file that will be uploaded.
:return: True if upload works and False if something went wrong.
:rtype: Boolean
"""
try:
blob = BlobClient.from_connection_string(conn_str=connection, container_name=container_name, blob_name=local_file_name)
with open(full_path,'rb') as data:
blob.upload_blob(data)
print('File {0} uploaded'.format(local_file_name))
return True
except ValueError:
print("Something went wrong")
return False
async def _test_auth_connection_string_async(self):
# [START auth_from_connection_string]
from azure.storage.blob.aio import BlobServiceClient
blob_service_client = BlobServiceClient.from_connection_string(self.connection_string)
# [END auth_from_connection_string]
# [START auth_from_connection_string_container]
from azure.storage.blob.aio import ContainerClient
container_client = ContainerClient.from_connection_string(
self.connection_string, container_name="mycontainer")
# [END auth_from_connection_string_container]
# [START auth_from_connection_string_blob]
from azure.storage.blob.aio import BlobClient
blob_client = BlobClient.from_connection_string(
self.connection_string, container_name="mycontainer", blob_name="blobname.txt")
# [END auth_from_connection_string_blob]
# Get account information for the Blob Service
account_info = await blob_service_client.get_account_information()
assert account_info is not None
def download_blob_file(connection,container_name,blob_name,path_output):
"""
Get blob file from cloud and downloads on local machine.
:param Object connection: Azure connection instance
:param str container_name: Azure Container name.
:param str blob_name: Azure Blob name to download.
:param str blob_name: Path where file will be saved.
:return: True if download works and False if something went wrong.
:rtype: Boolean
"""
try:
blob = BlobClient.from_connection_string(conn_str=connection, container_name=container_name, blob_name=blob_name)
with open(path_output,'wb') as myblob:
blob_data = blob.download_blob()
blob_data.readinto(myblob)
print('File saved in {0}'.format(path_output))
return True
except ValueError:
print("Something went wrong")
return False