def main(myblob: func.InputStream):
#//TODO: Ensure queue name is 'weather-data'
queue_name = 'weather-data'
#//TODO: Add Storage Account Name and Key from https://portal.azure.com/#@[user_email]/resource/subscriptions/[subscription_id]/resourceGroups/[resource_group_name]/providers/Microsoft.Storage/storageAccounts/[storage_account_name]/keys
block_blob_service = BlockBlobService(account_name='//TODO: ', account_key=' //TODO')
queue_service = QueueService(account_name='//TODO', account_key='//TODO')
queue_service.encode_function = QueueMessageFormat.text_base64encode
file_name = myblob.name.split("/")[1]
#// Ensure that files are added to a blob container named 'weather-data'
block_blob_service.get_blob_to_path('weather-data', file_name, file_name)
with open(file_name, "r+") as file:
reader = csv.reader(file)
for idx, data in enumerate(reader):
if idx != 0:
if(len(data)> 13):
city, country = data[6].split(",")
datapoint =json.dumps({"date": data[1],"city": city, "country": country, "temperature": data[13][:-2]})
queue_service.put_message(queue_name, datapoint)
else:
logging.info(len(data))
logging.info(f"Python blob trigger function processed blob \n"
f"Name: {myblob.name}\n"
f"Blob Size: {myblob.length} bytes\n"
)
class AzureProvider(BaseProvider):
"""Queue Provider for the Microsoft Azure."""
def __init__(self, account_name, account_key):
self.queue_service = QueueService(
account_name=account_name, account_key=account_key
)
def put_message(self, queue_name: str, message: str):
self.queue_service.put_message(
queue_name=queue_name,
message_text=message,
)
def get_message(self, queue_name: str):
try:
queue_message = self.queue_service.get_messages(
queue_name=queue_name, numofmessages=1
).queue_messages[0]
except IndexError:
return {}
else:
return {
attr_name: getattr(queue_message, attr_name)
for attr_name in dir(queue_message)
if not attr_name.startswith('_')
}
def delete_message(self, queue_name: str, message: dict):
message_id = message.get('message_id')
pop_receipt = message.get('pop_receipt')
self.queue_service.delete_message(
queue_name, message_id=message_id, popreceipt=pop_receipt,
)
class AzureQueue:
def __init__(self,accountName,key):
"""
Initialize a new connection to azure queue stoage
@param accountName : the storage account in azure
@param key : the api key
"""
self._accountName = accountName
self._key = key
self._innerService = QueueService(account_name=self._accountName, account_key=self._key)
def push_message(self,queue,message):
"""
Enqueue a new message into the given queue
@param queue : the queue name
¶m message : the message to put into the queue
"""
if not queue:
raise ValueError("queue name is mandatory ")
if not message:
raise ValueError("message is mandatory")
self._innerService.put_message(queue,message)
def render_video(request):
template = loader.get_template('app/render_video.html')
vidstatus = 'No Video Found.'
queue_service = QueueService(account_name=os.environ['SVPD_STORAGE_ACCOUNT_NAME'], account_key=os.environ['SVPD_STORAGE_ACCOUNT_KEY'])
messages = queue_service.get_messages(os.environ['SVPD_STORAGE_ACCOUNT_READY_TO_ENCODE'], num_messages=1, visibility_timeout=1*60)
for message in messages:
vidstatus = 'Queued for Rendering: ' + message.content
message_obj = json.loads(message.content)
access_token = ams_authenticate()['access_token']
asset = ams_post_request(access_token, "Assets", {
'Name': message_obj['filename'],
'AlternateId': message_obj['folder']})
asset_container = urllib.parse.urlparse(asset['Uri']).path[1:]
asset_file = ams_post_request(access_token, "Files", {
'IsEncrypted': 'false',
'IsPrimary': 'false',
'MimeType': 'video/mp4',
'ContentFileSize': message_obj['size'],
'Name': message_obj['filename'],
'ParentAssetId': asset['Id']})
block_blob_service = BlockBlobService(account_name=os.environ['SVPD_STORAGE_ACCOUNT_NAME'], account_key=os.environ['SVPD_STORAGE_ACCOUNT_KEY'])
from_url = block_blob_service.make_blob_url(os.environ['SVPD_STORAGE_ACCOUNT_UPLOADED'], message_obj['folder'] + '/' + message_obj['filename'])
block_blob_service.copy_blob(asset_container, message_obj['filename'], from_url)
job = ams_verbose_post_request(access_token, "Jobs", {
'Name': message_obj['filename'],
'InputMediaAssets': [{
'__metadata': { 'uri': os.environ['AMS_API_ENDPOINT'] + 'Assets(\'' + asset['Id'] + '\')' }
}],
'Tasks': [{
'Name': 'Adaptive Streaming Task',
'Configuration': 'Adaptive Streaming',
'MediaProcessorId': 'nb:mpid:UUID:ff4df607-d419-42f0-bc17-a481b1331e56',
'TaskBody': '<?xml version="1.0" encoding="utf-16"?><taskBody><inputAsset>JobInputAsset(0)</inputAsset><outputAsset assetCreationOptions="0" assetFormatOption="0" assetName="' + message_obj['filename'] + ' - MES v1.1" storageAccountName="' + os.environ['SVPD_STORAGE_ACCOUNT_NAME'] + '">JobOutputAsset(0)</outputAsset></taskBody>'
},{
'Name': 'Indexing Task',
'Configuration': '<?xml version="1.0" encoding="utf-8"?><configuration version="2.0"><input><metadata key="title" value="blah" /></input><settings></settings><features><feature name="ASR"><settings><add key="Language" value="English" /><add key="GenerateAIB" value="False" /><add key="GenerateKeywords" value="True" /><add key="ForceFullCaption" value="False" /><add key="CaptionFormats" value="ttml;sami;webvtt" /></settings></feature></features></configuration>',
'MediaProcessorId': 'nb:mpid:UUID:233e57fc-36bb-4f6f-8f18-3b662747a9f8',
'TaskBody': '<?xml version="1.0" encoding="utf-16"?><taskBody><inputAsset>JobInputAsset(0)</inputAsset><outputAsset assetCreationOptions="0" assetFormatOption="0" assetName="' + message_obj['filename'] + ' - Indexed" storageAccountName="' + os.environ['SVPD_STORAGE_ACCOUNT_NAME'] + '">JobOutputAsset(1)</outputAsset></taskBody>'
}]
})
queue_service.put_message(os.environ['SVPD_STORAGE_ACCOUNT_ENCODING'], json.dumps({
'filename': message_obj['filename'],
'folder': message_obj['folder'],
'size': message_obj['size'],
'job': job['d']}))
queue_service.delete_message(os.environ['SVPD_STORAGE_ACCOUNT_READY_TO_ENCODE'], message.id, message.pop_receipt)
return HttpResponse(template.render({
'vidstatus': vidstatus,
}, request))
class QueueBase(object):
"""docstring for QueueBase"""
def __init__(self):
super(QueueBase, self).__init__()
self.queue_service = QueueService(
account_name='boburstorage',
account_key=
'wRgukLsyhLtnI7qEk8mSGnIBC+IsiTTXEDF1/xnmBGDudJLSeYdtyuVzuSN5/cplJz88AJPyoVyjCmL9N1ECXw=='
)
def save_message_to_queue(self, queue, data):
self.queue_service.put_message(queue, data)
print('url added into queue...\n')
def get_data_from_url(self):
print('reading data from blob using url in queue...\n')
data = self.queue_service.get_messages('taskqueue')
response = urllib2.urlopen(data[0].content)
numbers = response.read()
self.queue_service.delete_message('taskqueue', data[0].id,
data[0].pop_receipt)
return numbers.splitlines()
def get_messages_from_queue(self, queue):
return self.queue_service.get_messages(queue)
def delete_message_from_queue(self, queue, message):
self.queue_service.delete_message(queue, message.id,
message.pop_receipt)
class AzureStorageQueue(Common.Contracts.Queue):
def __init__(self, queue_name, config: AzureStorageConfig):
self._queue_name = queue_name
self._queue_service = QueueService(account_name=config.account_name,
account_key=config.account_key)
self._queue_service.encode_function = QueueMessageFormat.text_base64encode
self._queue_service.decode_function = QueueMessageFormat.text_base64decode
def push(self, message):
self._queue_service.create_queue(self._queue_name)
self._queue_service.put_message(self._queue_name, message)
def pop(self):
self._queue_service.create_queue(self._queue_name)
messages = self._queue_service.peek_messages(self._queue_name)
for message in messages:
result = message.content
self._queue_service.delete_message(self._queue_name, message.id,
message.pop_receipt)
return result
def peek(self):
self._queue_service.create_queue(self._queue_name)
messages = self._queue_service.peek_messages(self._queue_name)
for message in messages:
return message.content
class QueueBase(object):
"""docstring for QueueBase"""
def __init__(self):
super(QueueBase, self).__init__()
self.queue_service = QueueService(
account_name='bobur',
account_key=
'6e60FZapOXAmUbFBw0SpE1lHRP3RkXOMYRaalWmRBoz4+xI5tvjaJzxXuYyt+yfWxjPXpz5X3PmyIFiQmSkjbw=='
)
def save_message_to_queue(self, queue, data):
self.queue_service.put_message(queue, data)
print('url added into queue...\n')
def get_data_from_url(self):
print('reading data from blob using url in queue...\n')
data = self.queue_service.get_messages('taskqueue')
response = urllib2.urlopen(data[0].content)
numbers = response.read()
self.queue_service.delete_message('taskqueue', data[0].id,
data[0].pop_receipt)
return numbers.splitlines()
def get_messages_from_queue(self, queue):
return self.queue_service.get_messages(queue)
def delete_message_from_queue(self, queue, message):
self.queue_service.delete_message(queue, message.id,
message.pop_receipt)
def enviar_aquivos_audio_blob(main_app, dir="audio_files/"):
for file in glob.glob(dir + "*.wav"):
try:
print("Processando arquivo " + file + "...")
meeting_code = file.split("_")[1].split("/")[1]
blob = meeting_code + "/" + file
print("Meeting code " + str(meeting_code))
blob_service = BlockBlobService(account_name=ACCOUNT_NAME,
account_key=ACCOUNT_KEY)
blob_service.create_blob_from_path(CONTAINER_NAME, blob, file)
if os.path.exists(file):
os.remove(file)
queue_service = QueueService(account_name=ACCOUNT_NAME,
account_key=ACCOUNT_KEY)
queue_service.encode_function = QueueMessageFormat.text_base64encode
payload = {
"meeting-code": meeting_code,
"blob": blob,
"file-name": util.get_file_with_extension(file)
}
payload = json.dumps(payload, ensure_ascii=False)
queue_service.put_message(QUEUE_NAME_AUDIO, payload)
print("Arquivo " + file + " processado com sucesso.")
main_app.mensagem["text"] = "File " + file + " synced successfully"
except:
traceback.format_exc()
class AzureQueue(object):
def __init__(self, queue_name):
self.conn = QueueService(account_name=os.getenv('AZURE_ACCOUNT_NAME'),
account_key=os.getenv('AZURE_ACCOUNT_KEY'))
self.queue_name = queue_name
self.conn.create_queue(queue_name)
self.conn.encode_function = QueueMessageFormat.binary_base64encode
self.conn.decode_function = QueueMessageFormat.binary_base64decode
def enqueue(self, func, *args, **kwargs):
task = SimpleTask(func, *args, **kwargs)
serialized_task = pickle.dumps(task, protocol=pickle.HIGHEST_PROTOCOL)
self.conn.put_message(self.queue_name, serialized_task)
return task.id
def dequeue(self):
messages = self.conn.get_messages(self.queue_name)
if len(messages) == 1:
serialized_task = messages[0]
task = pickle.loads(serialized_task.content)
self.conn.delete_message(self.queue_name, serialized_task.id,
serialized_task.pop_receipt)
return task
def get_length(self):
metadata = self.conn.get_queue_metadata(self.queue_name)
return metadata.approximate_message_count
class QueueStorageHandler(logging.Handler):
"""
Handler class which sends log messages to a Azure Storage queue.
"""
def __init__(self,
account_name=None,
account_key=None,
protocol='https',
queue='logs',
message_ttl=None,
visibility_timeout=None,
base64_encoding=False,
is_emulated=False,
):
"""
Initialize the handler.
"""
logging.Handler.__init__(self)
self.service = QueueService(account_name=account_name,
account_key=account_key,
is_emulated=is_emulated,
protocol=protocol)
self.meta = {'hostname': gethostname(), 'process': os.getpid()}
self.queue = _formatName(queue, self.meta)
self.queue_created = False
self.message_ttl = message_ttl
self.visibility_timeout = visibility_timeout
self.base64_encoding = base64_encoding
def emit(self, record):
"""
Emit a record.
Format the record and send it to the specified queue.
"""
try:
if not self.queue_created:
self.service.create_queue(self.queue)
self.queue_created = True
record.hostname = self.meta['hostname']
msg = self._encode_text(self.format(record))
self.service.put_message(self.queue,
msg,
self.visibility_timeout,
self.message_ttl)
except (KeyboardInterrupt, SystemExit):
raise
except:
self.handleError(record)
def _encode_text(self, text):
if self.base64_encoding:
text = b64encode(text.encode('utf-8')).decode('ascii')
# fallback for the breaking change in azure-storage 0.33
elif sys.version_info < (3,):
if not isinstance(text, unicode):
text = text.decode('utf-8')
return text
def check():
queue_service = QueueService(account_name=account_name,
account_key=account_key)
queue_service.create_queue('monitoring')
while True:
print("Adding new user to scrap...")
queue_service.put_message('monitoring',
f'monitoring_{int(time.time())}')
time.sleep(10)
class QueueStorageHandler(logging.Handler):
"""
Handler class which sends log messages to a Azure Storage queue.
"""
def __init__(
self,
account_name=None,
account_key=None,
protocol='https',
queue='logs',
message_ttl=None,
visibility_timeout=None,
base64_encoding=False,
is_emulated=False,
):
"""
Initialize the handler.
"""
logging.Handler.__init__(self)
self.service = QueueService(account_name=account_name,
account_key=account_key,
is_emulated=is_emulated,
protocol=protocol)
self.meta = {'hostname': gethostname(), 'process': os.getpid()}
self.queue = _formatName(queue, self.meta)
self.queue_created = False
self.message_ttl = message_ttl
self.visibility_timeout = visibility_timeout
self.base64_encoding = base64_encoding
def emit(self, record):
"""
Emit a record.
Format the record and send it to the specified queue.
"""
try:
if not self.queue_created:
self.service.create_queue(self.queue)
self.queue_created = True
record.hostname = self.meta['hostname']
msg = self._encode_text(self.format(record))
self.service.put_message(self.queue, msg, self.visibility_timeout,
self.message_ttl)
except (KeyboardInterrupt, SystemExit):
raise
except:
self.handleError(record)
def _encode_text(self, text):
if self.base64_encoding:
text = b64encode(text.encode('utf-8')).decode('ascii')
# fallback for the breaking change in azure-storage 0.33
elif sys.version_info < (3, ):
if not isinstance(text, unicode):
text = text.decode('utf-8')
return text
def writeToAzureQueue(outputList):
# send the output to the queue
logging.info("outputqueueLength = %s" % (str(len(outputList))))
connect_str = os.getenv("AzureWebJobsStorage")
queue_name = os.getenv("glossaryOutPutQueue")
queue_service = QueueService(connection_string=connect_str)
queue_service.encode_function = QueueMessageFormat.binary_base64encode
queue_service.decode_function = QueueMessageFormat.binary_base64decode
queue_service.put_message(queue_name,
json.dumps(outputList).encode('utf-8'))
class AzureStorageQueue(Queue):
"""Interface for interacting with an Azure Storage Queue (through the Queue
contract)"""
def __init__(self, queue_name, config: AzureStorageConfig):
"""Initializes the storage queue.
:param queue_name: The name of the queue to access.
If a queue with this name doesn't already exist on the
storage account, the queue will be created on the first operation.
:param config: AzureStorageConfig with a valid account name and
account key
"""
self._queue_name = queue_name
self._queue_service = QueueService(account_name=config.account_name,
account_key=config.account_key)
self._queue_service.encode_function = \
QueueMessageFormat.text_base64encode
self._queue_service.decode_function = \
QueueMessageFormat.text_base64decode
def push(self, message):
"""Pushes a new message onto the queue."""
self._queue_service.create_queue(self._queue_name)
self._queue_service.put_message(self._queue_name, message)
def pop(self):
"""Pops the first message from the queue and returns it."""
self._queue_service.create_queue(self._queue_name)
# get_messages prevents another client from getting the message
# before we've had a chance to delete it. The visibility_timeout
# prevents the message from being seen by other clients
# for X seconds.
messages = self._queue_service.get_messages(self._queue_name,
visibility_timeout=30)
for message in messages:
result = message.content
self._queue_service.delete_message(self._queue_name, message.id,
message.pop_receipt)
return result
def peek(self):
"""Peeks the fist message from the queue and returns it."""
self._queue_service.create_queue(self._queue_name)
messages = self._queue_service.peek_messages(self._queue_name)
for message in messages:
return message.content
@staticmethod
def create(queue_name: str):
"""Helper function for creating a Azure Storage Queue from the
storage_config property defined inside of AzureConfig."""
azure_config = AzureConfig()
return AzureStorageQueue(queue_name, azure_config.storage_config)
def get_new_proxy():
queue_service = QueueService(account_name=account_name,
account_key=account_key)
messages = queue_service.get_messages('proxies')
if messages:
for message in messages:
new_proxy = message.content
queue_service.delete_message('proxies', message.id,
message.pop_receipt)
queue_service.put_message('proxies', new_proxy)
return new_proxy
def detail(request, question_id):
queue_service = QueueService(account_name='firstsep9ed1', account_key='W+XOdAGrNzjtqTjCopNYZ5wX09Rfy1MTNLGwGnza6eofPkVqXMePyC5ovA+rkd3m4nDxEJeYcY0wCEcodLxyWQ==')
#queue_service.create_queue('taskqueue')
#message = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
message = base64.b64encode(bytes(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), 'utf-8'))
queue_service.put_message('test-python-queue', message.decode('ascii'))
try:
question = Question.objects.get(pk=question_id)
except Question.DoesNotExist:
raise Http404("Question does not exist")
return render(request, 'polls/detail.html', {'question': question})
class AzureInvoker(object):
def __init__(self, account, key, queue_name):
self.queue_service = QueueService(account_name=account,
account_key=key)
self.queue_service.encode_function = QueueMessageFormat.text_base64encode
self.queue = queue_name
self.TIME_LIMIT = True
def invoke(self, payload):
self.queue_service.put_message(
self.queue,
json.dumps(payload, ensure_ascii=False).decode('utf-8'))
return {}
def config(self):
return {"function_name": self.queue}
def main(myblob: func.InputStream):
logging.info(
f"Python blob trigger function processed blob \n"
f"Name: {myblob.name[len(os.environ['remoteStorageInputContainer'])+1:]}\n"
f"Blob Size: {myblob.length} bytes")
name = myblob.name[len(os.environ['remoteStorageInputContainer']) + 1:]
logging.info(name)
queue_service = QueueService(
account_name=os.environ['remoteStorageAccountName'],
account_key=os.environ['remoteStorageAccountKey'])
queue_service.encode_function = QueueMessageFormat.text_base64encode
now = datetime.strftime(datetime.now(), "%Y-%m-%dT%H:%M:%S%Z")
video = "{\"filename\":\"" + name + "\", \"location\":\"Utrecht\", \"track\":\"5b\", \"timestamp\":\"" + now + "\"}"
#video_base64 = base64.b64encode(bytes(video, 'utf-8'))
queue_service.put_message(os.environ['AzureQueueName'], video)
class WorkloadTracker(object):
"""
Dedicated class to track important events during the running of the workload.
"""
def __init__(self, logger):
self.config = Config()
self.logger = logger
self.init_storage_services()
def init_storage_services(self):
"""
Initializes the storage service clients using values from config.py.
:return: True on success. False on failure.
:rtype: boolean
"""
try:
# creates instances of Azure QueueService
self.workload_queue_service = QueueService(
account_name=self.config.storage_account_name,
sas_token=self.config.workload_tracker_sas_token)
self.workload_queue_service.create_queue(
self.config.workload_tracker_queue_name)
self.workload_queue_service.encode_function = models.QueueMessageFormat.noencode
return True
except Exception as ex:
self.logger.Exception(ex, self.init_storage_services.__name__)
return False
def write(self, event_type, content=None):
"""
Write the event to the dedicated workload tracker queue
"""
# create an event
evt = WorkloadEvent()
evt.event_type = int(event_type)
evt.content = content
# write serialized event to Azure queue
serialized_event = json.dumps(evt.__dict__)
self.workload_queue_service.put_message(
self.config.workload_tracker_queue_name, serialized_event)
def send_tweet(tweet_message, in_reply_to, entity):
queue_name = settings.AZURE_QUEUE_NAME
queue_service = QueueService(
account_name=settings.TWITTERBOT_STORAGE_ACCOUNT_NAME,
account_key=settings.TWITTERBOT_STORAGE_ACCOUNT_KEY)
queue_service.create_queue(queue_name)
queue_message = {
'id': entity['id'],
'tweet': {
'status': tweet_message,
'in_reply_to_status_id': in_reply_to
},
'percentiles': entity['percentiles']
}
queue_service.put_message(
queue_name,
base64.b64encode(
json.dumps(queue_message).encode('utf-8')).decode('utf-8'))
def queue():
account_name = config.STORAGE_ACCOUNT_NAME
account_key = config.STORAGE_ACCOUNT_KEY
queue_service = QueueService(account_name=account_name,
account_key=account_key)
print("Creating task queue")
task_queue_name = config.TASK_QUEUE_NAME
queue_service.create_queue(task_queue_name)
print("Task queue created")
queue_service.put_message(task_queue_name, u'message1')
messages = queue_service.get_messages(task_queue_name, num_messages=16)
for message in messages:
print(message.content)
queue_service.delete_message(task_queue_name, message.id,
message.pop_receipt)
def generate(id, n):
global TERMINATE
queue_service = QueueService(account_name='izastorm',
account_key='isUic1EEbXg8l53zUrl+o1Jmf8JPze/E8S5XQ3ActlrmpEmGqMSKdkSP/RTF4aFAdQmLeVy6DWT3pGJ1k/I2HA==')
dateFormat = '%Y-%m-%d-%H-%M-%S'
time.sleep(random.uniform(0.0, 5.0))
while(True):
data ={}
datetimeValues = {}
datetimeValues['format'] = dateFormat
datetimeValues['source'] = 'RTC_DS1307'
#datetimeValues['stationID'] = random.randint(0,10)
datetimeValues['stationID'] = id
datetimeValues['value'] = time.strftime(dateFormat)
sensorsValues = {}
sensorsValues['LDR'] = random.uniform(0.0,100.0)
sensorsValues['DHT22_AH'] = random.uniform(0.0, 100.0)
sensorsValues['BMP085_PRESSURE'] = random.uniform(90000.0, 110000.0)
sensorsValues['DHT22_TEMP'] = random.uniform(0.0, 45.0)
data['sensors'] = sensorsValues
data['datetime'] = datetimeValues
#print json.dumps(data).decode('utf-8')
try:
queue_service.put_message('stormtest', json.dumps(data).decode('utf-8'))
except:
time.sleep(random.uniform(0.0, 2.0))
try:
time.sleep(n)
except KeyboardInterrupt:
TERMINATE = False
if TERMINATE:
exit(0)
def save_dic_to_blob(vid_id):
# save dic as blob
account_name = 'ctrlfvfunctionaa670'
account_key = 'MoPjP9rLlfN8nK4+uejH6fSCwZHOqqvvfwVa6Ais3emwtGlly59oCS2Z8VQ+8OiKzzVwMghRImUPddVyMPAN9Q=='
corpus_seg_container_name = "corpus-segments-container"
blob_name = vid_id + ".txt"
print("saving dic as blob...")
block_blob_service = BlockBlobService(account_name, account_key)
block_blob_service.create_blob_from_text(
corpus_seg_container_name, blob_name,
json.dumps(list(_time_transcript_dic.items())))
# add message to asr-to-CorpusSegMerger queue
queue_service = QueueService(account_name=account_name,
account_key=account_key)
queue_name = "asr-to-corpus-seg-merger-q"
print('Creating message for queue:' + queue_name)
message = {"ID": blob_name}
message = json.dumps(message)
message = base64.b64encode(message.encode("ascii")).decode()
queue_service.put_message(queue_name, message)
print("Sent message:" + message)
def run_temperature():
queue_service = QueueService(account_name, account_key)
queue1 = "NormalTemp"
queue2 = "HighTemp"
queue_service.create_queue(queue1)
print("Queue created successfully...")
queue_service.create_queue(queue2)
print("Queue created successfully...")
for i in range(1, 10):
temp = generateTemp()
queue_service.put_message(queue1, temp)
if(temp > 80):
# Code for alert function will be plcaed here
tkMessageBox.showinfo(message=temp"is the temperature")
queue_service.put_message(queue2, temp)
class TimeLineWorker:
QUEUE_KEY = ''
ACCOUNT_NAME = ''
QUEUE_NAME = ''
def __init__(self):
self.queue_service = QueueService(account_name=TimeLineWorker.ACCOUNT_NAME,
account_key=TimeLineWorker.QUEUE_KEY)
def insert_message(self):
obj = {
"message": "test message",
"other_key": 10
}
message = unicode(json.dumps(obj))
self.queue_service.put_message(TimeLineWorker.QUEUE_NAME, message)
def get_next_message(self):
messages = self.queue_service.get_messages(TimeLineWorker.QUEUE_NAME)
for message in messages:
print message.content
self.queue_service.delete_message(TimeLineWorker.QUEUE_NAME, message.id, message.pop_receipt)
def upload_file(request):
template = loader.get_template('app/upload_file.html')
if request.method == 'POST' and request.FILES['myfile']:
myfile = request.FILES['myfile']
folder = datetime.datetime.now().strftime('%Y%m%d/')+str(uuid.uuid4())
filename = folder + '/' + myfile.name
# save the file to Azure Storage
block_blob_service = BlockBlobService(account_name=os.environ['SVPD_STORAGE_ACCOUNT_NAME'], account_key=os.environ['SVPD_STORAGE_ACCOUNT_KEY'])
block_blob_service.create_blob_from_bytes(os.environ['SVPD_STORAGE_ACCOUNT_UPLOADED'], filename, myfile.read())
# put a message into a queue letting the system know the video is ready for processing
queue_service = QueueService(account_name=os.environ['SVPD_STORAGE_ACCOUNT_NAME'], account_key=os.environ['SVPD_STORAGE_ACCOUNT_KEY'])
queue_service.put_message(os.environ['SVPD_STORAGE_ACCOUNT_READY_TO_ENCODE'], json.dumps({
'filename': myfile.name,
'folder': folder,
'size': str(myfile.size)}))
return HttpResponse(template.render({
'uploaded_file_name': filename,
}, request))
return HttpResponse(template.render({ }, request))
class AzureQueue(object):
def __init__(self, account_name, account_key, queue_name):
self.queue_name = queue_name
self.queue_service = QueueService(account_name=account_name, account_key=account_key)
self.queue_service.create_queue(self.queue_name)
def put_message_into_queue(self, content) -> QueueMessage:
"""
Publishes a message with `content`
:param content: The queue message
:returns: A QueueMessage that has the message as well as metadata
:rtype: QueueMessage
"""
return self.queue_service.put_message(self.queue_name, content)
def get_messages(self) -> list:
"""
Retrieves all of the messages that have been published into queue
:param content: The queue message
:returns: List of Queue messages
:rtype: list
"""
return self.queue_service.get_messages(self.queue_name)
def delete_message_from_queue(self, message_id, pop_receipt):
self.queue_service.delete_message(self.queue_name, message_id, pop_receipt)
def get_message_count(self):
queue_metadata = self.queue_service.get_queue_metadata(self.queue_name)
return queue_metadata.approximate_message_count
def delete(self):
return self.queue_service.delete_queue(self.queue_name)
def empty(self):
messages = queue_service.get_messages(self.queue_name,
num_messages=BATCH_NUMBER,
visibility_timeout=TIMEOUT_IN_SECONDS)
for message in messages:
self.queue_service.delete_message(self.queue_name, message.id, message.pop_receipt)
def test_sas_add(self):
# SAS URL is calculated from storage key, so this test runs live only
if TestMode.need_recordingfile(self.test_mode):
return
# Arrange
token = self.qs.generate_shared_access_signature(
self.test_queues[0],
self._get_shared_access_policy(QueueSharedAccessPermissions.ADD),
)
# Act
service = QueueService(
account_name=self.settings.STORAGE_ACCOUNT_NAME,
sas_token=token,
)
self._set_service_options(service, self.settings)
result = service.put_message(self.test_queues[0], 'addedmessage')
# Assert
result = self.qs.get_messages(self.test_queues[0])
self.assertEqual('addedmessage', result[0].message_text)
def test_sas_add(self):
# SAS URL is calculated from storage key, so this test runs live only
if TestMode.need_recordingfile(self.test_mode):
return
# Arrange
token = self.qs.generate_shared_access_signature(
self.test_queues[0],
self._get_shared_access_policy(QueueSharedAccessPermissions.ADD),
)
# Act
service = QueueService(
account_name=self.settings.STORAGE_ACCOUNT_NAME,
sas_token=token,
)
self._set_service_options(service, self.settings)
result = service.put_message(self.test_queues[0], 'addedmessage')
# Assert
result = self.qs.get_messages(self.test_queues[0])
self.assertEqual('addedmessage', result[0].message_text)
def test_sas_add(self):
# SAS URL is calculated from storage key, so this test runs live only
if TestMode.need_recordingfile(self.test_mode):
return
# Arrange
queue_name = self._create_queue()
token = self.qs.generate_queue_shared_access_signature(
queue_name,
QueuePermissions.ADD,
datetime.utcnow() + timedelta(hours=1),
)
# Act
service = QueueService(
account_name=self.settings.STORAGE_ACCOUNT_NAME,
sas_token=token,
)
self._set_service_options(service, self.settings)
result = service.put_message(queue_name, u'addedmessage')
# Assert
result = self.qs.get_messages(queue_name)
self.assertEqual(u'addedmessage', result[0].content)
def test_sas_add(self):
# SAS URL is calculated from storage key, so this test runs live only
if TestMode.need_recording_file(self.test_mode):
return
# Arrange
queue_name = self._create_queue()
token = self.qs.generate_queue_shared_access_signature(
queue_name,
QueuePermissions.ADD,
datetime.utcnow() + timedelta(hours=1),
)
# Act
service = QueueService(
account_name=self.settings.STORAGE_ACCOUNT_NAME,
sas_token=token,
)
self._set_service_options(service, self.settings)
result = service.put_message(queue_name, u'addedmessage')
# Assert
result = self.qs.get_messages(queue_name)
self.assertEqual(u'addedmessage', result[0].content)
# Main thread - Check if port 25565 and move to the next if it's not. Once a free port has been found, create a new instance (up to 3)
while instanceCreated == 0:
freeport = checkPort(defaultport)
if freeport == 0:
defaultport += 1
counter += 1
if counter > maxInstances:
print "Too many instances! you can't run more than %s instances" % maxInstances
sys.exit()
else:
if debug == 1: print "Port %s is free" % defaultport
container = createMcInstance(defaultport)
instanceCreated = 1
counter +=1
# Return Instance ID, port and total number of instances once finished creating the server
print "Instance ID: %s" % container["Id"]
print "Listening on port %s" % defaultport
print "There are %s instances running on this server" % counter
# Update servers queue
queue_service = QueueService(account_name, account_key)
try:
queue_service.create_queue(queuename)
except:
print "Queue creation failed."
queue_service.put_message(queuename, '{"ContainerID":"%s", "ContainerPort":"%s", "ServerID":"%s"}' % container["Id"], defaultport, socket.gethostname())
sys.exit()
# This is our initial state: the target image.
# Note that `scipy.optimize.fmin_l_bfgs_b` can only process flat vectors.
x = preprocess_image(target_image_path)
x = x.flatten()
for i in range(iterations):
print('Start of iteration', i)
start_time = time.time()
x, min_val, info = fmin_l_bfgs_b(evaluator.loss,
x,
fprime=evaluator.grads,
maxfun=20)
print('Current loss value:', min_val)
# Save current generated image
img = x.copy().reshape((img_height, img_width, 3))
img = deprocess_image(img)
fname = result_prefix + '_at_iteration_%d.png' % i
#print(fname)
imsave(fname, img)
end_time = time.time()
print('Image saved as', fname)
print('Iteration %d completed in %ds' % (i, end_time - start_time))
# Generate image
imsave(output_image_path + output_image_name, img)
saveToBlob(output_image_name, output_image_path + output_image_name)
blob_image_url = 'https://imprinter.blob.core.windows.net/imprinted/' + output_image_name
queue_service.put_message('imprintresults', blob_image_url)
queue_service.delete_message(queue_name, message.id,
message.pop_receipt)
def main(req: func.HttpRequest) -> func.HttpResponse:
logging.info('Python HTTP trigger function processed a request.')
# DefaultAzureCredential supports managed identity or environment configuration (see docs)
credential = DefaultAzureCredential()
# parse parameters
storage_account_source = os.environ["par_storage_account_name_source"]
storage_account_source_url = "https://" + storage_account_source + ".blob.core.windows.net"
storage_account_backup = os.environ["par_storage_account_name_backup"]
storage_account_backup_url = "https://" + storage_account_backup + ".blob.core.windows.net"
# create blob client for backup and source
credential = DefaultAzureCredential()
client_source = BlobServiceClient(account_url=storage_account_source_url,
credential=credential)
client_backup = BlobServiceClient(account_url=storage_account_backup_url,
credential=credential)
# Create queue clients
queue_service = QueueService(
account_name=os.environ['par_storage_account_name_queue'],
account_key=os.environ['par_storage_account_key_queue'])
queue_service.encode_function = QueueMessageFormat.text_base64encode
# Get all blobs in sourcecontainer
container_source_list = client_source.list_containers()
for container in container_source_list:
# Log container name
logging.info(container.name)
container_source = client_source.get_container_client(container.name)
# Get all blobs in container
prev_blob_name = ""
prev_blob_etag = ""
blob_source_list = container_source.list_blobs(include=['snapshots'])
for blob in blob_source_list:
if blob.snapshot == None:
# Blob that is not snapshot.
# 1. Check if snapshot needs to be created
if prev_blob_name != blob.name:
# New blob without snapshot, create snapshot/backup
logging.info("new blob" + blob.name +
", create snapshot/backup")
create_snapshot(client_source, queue_service,
container.name, blob.name, blob.etag)
elif prev_blob_etag != blob.etag:
# Existing blob that has changed, create snapshot/backup
logging.info(blob.name +
"has changed, create snapshot/backup")
create_snapshot(client_source, queue_service,
container.name, blob.name, blob.etag)
# 2. Check if incremental backup needs to be created
# get blob backup and source properties
blob_source = client_source.get_blob_client(
container=container.name, blob=blob.name)
source_last_modified = blob_source.get_blob_properties(
)['last_modified']
source_etag = str(
blob_source.get_blob_properties()['etag']).replace(
"\"", "")
blob_name_backup = append_timestamp_etag(
blob.name, source_last_modified, source_etag)
blob_backup = client_backup.get_blob_client(
container=container.name + "bak", blob=blob_name_backup)
blob_exists = check_blob_exists(blob_backup)
# Check if blob exists
if blob_exists == False:
# Latest blob does not yet exist in backup, create message on queue to update
queue_json = "{" + "\"container\":\"{}\", \"blob_name\":\"{}\", \"etag\":\"{}\"".format(
container.name, blob.name, source_etag) + "}"
logging.info("backup needed for: " + queue_json)
queue_service.put_message(os.environ['par_queue_name'],
queue_json)
#asyncio.run(copy_adf_blob_source_backup(blob_source, blob_backup))
prev_blob_name = blob.name
prev_blob_etag = blob.etag
result = {"status": "ok"}
return func.HttpResponse(str(result))
#!/usr/bin/python import sys from azure.storage.queue import QueueService account_name = "mcdockerqueue" queuename = sys.argv[1] account_key = sys.argv[2] queue_service = QueueService(account_name, account_key) try: queue_service.create_queue(queuename) except: print "Queue creation failed." for i in range(0,4): queue_service.put_message(queuename, 'Hello World %s' %i) print i
import numpy as np
import json, base64, time
from azure.storage.queue import QueueService, QueueMessageFormat
########################################################################################################################
# Credentials
queue_service = QueueService(account_name='', account_key='')
queue_service.encode_function = QueueMessageFormat.text_base64encode
########################################################################################################################
# Job parameters
container = 'seismic'
partial_gradient_path = 'partial_gradients/'
full_gradient_path = 'full_gradients/'
gradient_name = 'test_grad_'
iteration = 1
maxiter = 3
batchsize = 100
# Encode msg and submit job
msg = container + '&' + partial_gradient_path + '&' + full_gradient_path + '&' + gradient_name + '&' + str(
iteration) + '&' + str(maxiter) + '&' + str(batchsize)
queue_service.put_message('iterationqueue', msg)
class AzureService(object):
VISIBILITY_TIMEOUT = 5*60
def __init__(self, connection_string, container_name, queue_get, queue_push, logger=None):
self.ctnname = container_name
self.getname = queue_get
self.pushname = queue_push
self.qs = QueueService(connection_string=connection_string,
protocol='https',
# endpoint_suffix='core.windows.net'
)
self.bs = BlockBlobService(connection_string=connection_string)
self.qs.create_queue(self.getname, timeout=1)
self.qs.create_queue(self.pushname, timeout=1)
self.bs.create_container(self.ctnname, timeout=1)
if logger: logger.info('Init Azure success')
def pushMessage(self, message, qname=None, logger=None):
if qname is None:
qname = self.pushname
try:
self.qs.put_message(self.pushname, message)
except Exception as e:
if logger:
logger.exception('ERROR PUSH MESSAGE ')
else:
print 'ERROR PUSH MESSAGE '
print e
def getMessage(self, qname=None, num=1, logger=None):
if qname is None:
qname = self.getname
try:
message = self.qs.get_messages(qname, num, visibility_timeout=self.VISIBILITY_TIMEOUT)
except Exception as e:
if logger:
logger.exception('ERROR GET MESSAGE ')
else:
print 'ERROR GET MESSAGE '
print e
return []
return message
def getReceiptInfo(self, logger=None):
message = self.getMessage(logger=logger)
if len(message) > 0:
rinfo = ReceiptSerialize.fromjson(message[0].content)
return message[0], rinfo
else:
return None, None
def count(self):
metadata_get = self.qs.get_queue_metadata(self.getname)
metadata_push = self.qs.get_queue_metadata(self.pushname)
generator = self.bs.list_blobs(self.ctnname)
bc = 0
for blob in generator:
bc += 1
return {'get_count' : metadata_get.approximate_message_count,
'push_count': metadata_push.approximate_message_count,
'blob_count': bc
}
def uploadFolder(self, folderpath, logger):
for filename in os.listdir(folderpath):
if len(filename) > 4:
suffix = filename[-4:].upper()
else:
continue
if '.JPG' == suffix or 'JPEG' == suffix:
receipt_metadata = ReceiptSerialize()
receipt_metadata.receiptBlobName = unicode(filename, 'utf-8')
self.qs.put_message(self.getname, b64encode(receipt_metadata.toString()).decode('utf-8'))
self.bs.create_blob_from_path(self.ctnname, receipt_metadata.receiptBlobName, os.path.join(folderpath, filename), max_connections=2, timeout=None)
logger.info('upload %s', filename)
def getImage(self, imgname, logger=None):
localpath= os.path.join(args.download_dir, imgname)
try:
self.bs.get_blob_to_path(self.ctnname, imgname, localpath)
except AzureMissingResourceHttpError as e:
if logger:
logger.error('Blob named ' + imgname + ' doesnot exist.' , exc_info=True)
else:
print 'Blob named ' + imgname + ' doesnot exist.'
print e
return ''
except Exception as e:
if logger:
logger.error('Exception while getting blob.', exc_info=True)
else:
print 'Exception while getting blob.'
print e
return None
return localpath
def deleteMessage(self, message, qname=None, logger=None):
if qname is None:
qname = self.getname
try:
self.qs.delete_message(qname, message.id, message.pop_receipt)
except Exception as e:
if logger:
logger.exception('ERROR DELETE MESSAGE ')
else:
print 'ERROR DELETE MESSAGE '
print e
def deleteImage(self, imgname, logger=None):
try:
self.bs.delete_blob(self.ctnname, imgname)
except Exception as e:
if logger:
logger.exception('ERROR DELETE IMAGE ')
else:
print 'ERROR DELETE IMAGE '
print e
def cleanUp(self):
count = 0
print('deleted: ')
while True:
messages = self.qs.get_messages(self.getname)
for message in messages:
count += 1
self.qs.delete_message(self.getname, message.id, message.pop_receipt)
if len(messages) == 0: break
print(str(count) + ' from queue-get')
count = 0
while True:
messages = self.qs.get_messages(self.pushname)
for message in messages:
count += 1
self.qs.delete_message(self.pushname, message.id, message.pop_receipt)
if len(messages) == 0: break
print(str(count) + ' from queue-push')
count = 0
generator = self.bs.list_blobs(self.ctnname)
for blob in generator:
count += 1
self.bs.delete_blob(self.ctnname, blob.name)
print(str(count) + ' from container')
def put_to_queue(name, content):
queue_service = QueueService(account_name=config.AZURE_STORAGE_NAME, account_key=config.AZURE_STORAGE_KEY)
queue_service.create_queue(name)
queue_service.put_message(name, content)
class Queue:
def __init__(self, account_name, account_key, queue_name="logqueue"):
"""Initialiaze a queue. The type is set by the
'ACS_LOGGING_QUEUE_TYPE' environment variable. If it is set to
'AzureStorageQueue' then values must be provided for
'account_name' and 'account_key' which are values associated
with the Azure Storage account. 'queue_name' is optional and
defaults to 'logqueue'.
"""
self.log = Log()
self.queue_type = config.ACS_LOGGING_QUEUE_TYPE
self.queue_name = queue_name
# self.log.debug("Queue type: " + self.queue_type + " / " + self.queue_name)
if self.queue_type == "AzureStorageQueue":
self.createAzureQueues(account_name, account_key)
elif self.queue_type == "LocalFile":
self.file_queue = open(config.UNPROCESSED_LOG_FILE, "w+")
else:
self.log.error("Unknown queue type: " + queue_type)
def createAzureQueues(self, account_name, account_key):
"""
Create a queue for unprocessed log messages. Entries in the queue
will be dequeued, processed and deleted upon success.
"""
self.queue_service = QueueService(account_name, account_key)
self.queue_service.create_queue(self.queue_name)
def close(self):
"""Perform any necessary clearnup on the queue
at the end of a run.
"""
if self.queue_type == "AzureStorageQueue":
pass
elif self.queue_type == "LocalFile":
self.file_queue.close()
else:
self.log.error("Unknown queue type: " + queue_type)
def enqueue(self, msg, level="INFO"):
msg = level + " - " + msg
if self.queue_type == "LocalFile":
file_queue.write(msg + "\n")
elif self.queue_type == "AzureStorageQueue":
self.queue_service.put_message(self.queue_name, msg)
self.log.debug(msg)
def dequeue(self):
messages = []
if self.queue_type == "LocalFile":
with open(config.UNPROCESSED_LOG_FILE, "r") as f:
messages = f.readlines()[1]
elif self.queue_type == "AzureStorageQueue":
messages = self.queue_service.get_messages(self.queue_name)
return messages
def delete(self, message):
self.queue_service.delete_message(self.queue_name, message.message_id, message.pop_receipt)
# with open(config.PROCESSED_LOG_FILE, 'a') as processed:
# processed.write(log)
# os.remove(config.UNPROCESSED_LOG_FILE)
def delete_queue(self, queue_name):
self.queue_service.delete_queue(queue_name, False)
def getLength(self):
"""
Get the approximate length of the queue
"""
queue_metadata = self.queue_service.get_queue_metadata(self.queue_name)
count = queue_metadata["x-ms-approximate-messages-count"]
return count
