def enviar_aquivos_audio_blob(main_app, dir="audio_files/"):
for file in glob.glob(dir + "*.wav"):
try:
print("Processando arquivo " + file + "...")
meeting_code = file.split("_")[1].split("/")[1]
blob = meeting_code + "/" + file
print("Meeting code " + str(meeting_code))
blob_service = BlockBlobService(account_name=ACCOUNT_NAME,
account_key=ACCOUNT_KEY)
blob_service.create_blob_from_path(CONTAINER_NAME, blob, file)
if os.path.exists(file):
os.remove(file)
queue_service = QueueService(account_name=ACCOUNT_NAME,
account_key=ACCOUNT_KEY)
queue_service.encode_function = QueueMessageFormat.text_base64encode
payload = {
"meeting-code": meeting_code,
"blob": blob,
"file-name": util.get_file_with_extension(file)
}
payload = json.dumps(payload, ensure_ascii=False)
queue_service.put_message(QUEUE_NAME_AUDIO, payload)
print("Arquivo " + file + " processado com sucesso.")
main_app.mensagem["text"] = "File " + file + " synced successfully"
except:
traceback.format_exc()
def main(myblob: func.InputStream):
#//TODO: Ensure queue name is 'weather-data'
queue_name = 'weather-data'
#//TODO: Add Storage Account Name and Key from https://portal.azure.com/#@[user_email]/resource/subscriptions/[subscription_id]/resourceGroups/[resource_group_name]/providers/Microsoft.Storage/storageAccounts/[storage_account_name]/keys
block_blob_service = BlockBlobService(account_name='//TODO: ', account_key=' //TODO')
queue_service = QueueService(account_name='//TODO', account_key='//TODO')
queue_service.encode_function = QueueMessageFormat.text_base64encode
file_name = myblob.name.split("/")[1]
#// Ensure that files are added to a blob container named 'weather-data'
block_blob_service.get_blob_to_path('weather-data', file_name, file_name)
with open(file_name, "r+") as file:
reader = csv.reader(file)
for idx, data in enumerate(reader):
if idx != 0:
if(len(data)> 13):
city, country = data[6].split(",")
datapoint =json.dumps({"date": data[1],"city": city, "country": country, "temperature": data[13][:-2]})
queue_service.put_message(queue_name, datapoint)
else:
logging.info(len(data))
logging.info(f"Python blob trigger function processed blob \n"
f"Name: {myblob.name}\n"
f"Blob Size: {myblob.length} bytes\n"
)
def writeToAzureQueue(outputList):
# send the output to the queue
logging.info("outputqueueLength = %s" % (str(len(outputList))))
connect_str = os.getenv("AzureWebJobsStorage")
queue_name = os.getenv("glossaryOutPutQueue")
queue_service = QueueService(connection_string=connect_str)
queue_service.encode_function = QueueMessageFormat.binary_base64encode
queue_service.decode_function = QueueMessageFormat.binary_base64decode
queue_service.put_message(queue_name,
json.dumps(outputList).encode('utf-8'))
def create_queue_service(account_name, account_key):
queue_service = None
try:
queue_service = QueueService(account_name=account_name,
account_key=account_key)
queue_service.encode_function = QueueMessageFormat.binary_base64encode
queue_service.decode_function = QueueMessageFormat.binary_base64decode
except Exception as e:
logging.error("Could not instantiate queue service: %s" % e)
return queue_service
def main(myblob: func.InputStream):
logging.info(
f"Python blob trigger function processed blob \n"
f"Name: {myblob.name[len(os.environ['remoteStorageInputContainer'])+1:]}\n"
f"Blob Size: {myblob.length} bytes")
name = myblob.name[len(os.environ['remoteStorageInputContainer']) + 1:]
logging.info(name)
queue_service = QueueService(
account_name=os.environ['remoteStorageAccountName'],
account_key=os.environ['remoteStorageAccountKey'])
queue_service.encode_function = QueueMessageFormat.text_base64encode
now = datetime.strftime(datetime.now(), "%Y-%m-%dT%H:%M:%S%Z")
video = "{\"filename\":\"" + name + "\", \"location\":\"Utrecht\", \"track\":\"5b\", \"timestamp\":\"" + now + "\"}"
#video_base64 = base64.b64encode(bytes(video, 'utf-8'))
queue_service.put_message(os.environ['AzureQueueName'], video)
def main(req: func.HttpRequest) -> func.HttpResponse:
logging.info('Python HTTP trigger function processed a request.')
# DefaultAzureCredential supports managed identity or environment configuration (see docs)
credential = DefaultAzureCredential()
# parse parameters
storage_account_source = os.environ["par_storage_account_name_source"]
storage_account_source_url = "https://" + storage_account_source + ".blob.core.windows.net"
storage_account_backup = os.environ["par_storage_account_name_backup"]
storage_account_backup_url = "https://" + storage_account_backup + ".blob.core.windows.net"
# create blob client for backup and source
credential = DefaultAzureCredential()
client_source = BlobServiceClient(account_url=storage_account_source_url,
credential=credential)
client_backup = BlobServiceClient(account_url=storage_account_backup_url,
credential=credential)
# Create queue clients
queue_service = QueueService(
account_name=os.environ['par_storage_account_name_queue'],
account_key=os.environ['par_storage_account_key_queue'])
queue_service.encode_function = QueueMessageFormat.text_base64encode
# Get all blobs in sourcecontainer
container_source_list = client_source.list_containers()
for container in container_source_list:
# Log container name
logging.info(container.name)
container_source = client_source.get_container_client(container.name)
# Get all blobs in container
prev_blob_name = ""
prev_blob_etag = ""
blob_source_list = container_source.list_blobs(include=['snapshots'])
for blob in blob_source_list:
if blob.snapshot == None:
# Blob that is not snapshot.
# 1. Check if snapshot needs to be created
if prev_blob_name != blob.name:
# New blob without snapshot, create snapshot/backup
logging.info("new blob" + blob.name +
", create snapshot/backup")
create_snapshot(client_source, queue_service,
container.name, blob.name, blob.etag)
elif prev_blob_etag != blob.etag:
# Existing blob that has changed, create snapshot/backup
logging.info(blob.name +
"has changed, create snapshot/backup")
create_snapshot(client_source, queue_service,
container.name, blob.name, blob.etag)
# 2. Check if incremental backup needs to be created
# get blob backup and source properties
blob_source = client_source.get_blob_client(
container=container.name, blob=blob.name)
source_last_modified = blob_source.get_blob_properties(
)['last_modified']
source_etag = str(
blob_source.get_blob_properties()['etag']).replace(
"\"", "")
blob_name_backup = append_timestamp_etag(
blob.name, source_last_modified, source_etag)
blob_backup = client_backup.get_blob_client(
container=container.name + "bak", blob=blob_name_backup)
blob_exists = check_blob_exists(blob_backup)
# Check if blob exists
if blob_exists == False:
# Latest blob does not yet exist in backup, create message on queue to update
queue_json = "{" + "\"container\":\"{}\", \"blob_name\":\"{}\", \"etag\":\"{}\"".format(
container.name, blob.name, source_etag) + "}"
logging.info("backup needed for: " + queue_json)
queue_service.put_message(os.environ['par_queue_name'],
queue_json)
#asyncio.run(copy_adf_blob_source_backup(blob_source, blob_backup))
prev_blob_name = blob.name
prev_blob_etag = blob.etag
result = {"status": "ok"}
return func.HttpResponse(str(result))
import numpy as np
import json, base64, time
from azure.storage.queue import QueueService, QueueMessageFormat
########################################################################################################################
# Credentials
queue_service = QueueService(account_name='', account_key='')
queue_service.encode_function = QueueMessageFormat.text_base64encode
########################################################################################################################
# Job parameters
container = 'seismic'
partial_gradient_path = 'partial_gradients/'
full_gradient_path = 'full_gradients/'
gradient_name = 'test_grad_'
iteration = 1
maxiter = 3
batchsize = 100
# Encode msg and submit job
msg = container + '&' + partial_gradient_path + '&' + full_gradient_path + '&' + gradient_name + '&' + str(
iteration) + '&' + str(maxiter) + '&' + str(batchsize)
queue_service.put_message('iterationqueue', msg)
def main(req: func.HttpRequest) -> func.HttpResponse:
logging.info('Python HTTP trigger function processed a request.')
user_name = req.params.get('userName')
if not user_name:
return func.HttpResponse(
status_code=401,
headers=DEFAULT_RETURN_HEADER,
body=json.dumps({"error": "invalid userName given or omitted"})
)
try:
req_body = req.get_json()
logging.debug(req.get_json())
storage_account = req_body["storageAccount"]
storage_account_key = req_body["storageAccountKey"]
storage_container = req_body["storageContainer"]
except ValueError:
return func.HttpResponse(
"ERROR: Unable to decode POST body",
status_code=400
)
if not storage_container or not storage_account or not storage_account_key:
return func.HttpResponse(
"ERROR: storage container/account/key/queue not specified.",
status_code=401
)
# Create blob service for storage account (retrieval source)
blob_service = BlockBlobService(
account_name=storage_account,
account_key=storage_account_key)
# Queue service for perm storage and queue
queue_service = QueueService(
account_name=os.getenv('STORAGE_ACCOUNT_NAME'),
account_key=os.getenv('STORAGE_ACCOUNT_KEY')
)
queue_service.encode_function = QueueMessageFormat.text_base64encode
try:
blob_list = []
for blob_object in blob_service.list_blobs(storage_container):
blob_url = URL(
blob_service.make_blob_url(
storage_container,
blob_object.name
)
)
# Check for supported image types here.
if ImageFileType.is_supported_filetype(blob_url.suffix):
logging.debug("INFO: Building sas token for blob " + blob_object.name)
# create sas signature
sas_signature = blob_service.generate_blob_shared_access_signature(
storage_container,
blob_object.name,
BlobPermissions.READ,
datetime.utcnow() + timedelta(hours=1)
)
logging.debug("INFO: have sas signature {}".format(sas_signature))
signed_url = blob_url.with_query(sas_signature)
blob_list.append(signed_url.as_uri())
logging.debug("INFO: Built signed url: {}".format(signed_url))
msg_body = {
"imageUrl": signed_url.as_uri(),
"fileName": str(blob_url.name),
"fileExtension": str(blob_url.suffix),
"directoryComponents": get_filepath_from_url(blob_url, storage_container),
"userName": user_name
}
body_str = json.dumps(msg_body)
queue_service.put_message("onboardqueue", body_str)
else:
logging.info("Blob object not supported. Object URL={}".format(blob_url.as_uri))
return func.HttpResponse(
status_code=202,
headers=DEFAULT_RETURN_HEADER,
body=json.dumps(blob_list)
)
except Exception as e:
logging.error("ERROR: Could not build blob object list. Exception: " + str(e))
return func.HttpResponse("ERROR: Could not get list of blobs in storage_container={0}. Exception={1}".format(
storage_container, e), status_code=500)
