def processMessage(data: DocumentData, startTime: datetime): startTime = datetime.utcnow() result = doMath() result = 0 args = sys.argv data.start = f'{startTime:%Y-%m-%d %H:%M:%S.%f%z}' data.host = args[2] data.port = args[4] data.worker = args[6] data.request = args[8] data.plataform = sys.platform data.node = platform.node() data.result = result data.version = "14" # Identifyier to filter logs # Calculating queue size storageKey = os.environ['StorageKey'] queue_service = QueueService(account_name='funcpyqueue2storage', account_key=storageKey) metadata = queue_service.get_queue_metadata('fibonaccicalculatorqueue') data.queuedMessages = metadata.approximate_message_count stopTime = datetime.utcnow() # Calculate times messageInsertionTime = datetime.strptime(data.msgInsertionTime, '%Y-%m-%d %H:%M:%S') data.stop = f'{stopTime:%Y-%m-%d %H:%M:%S.%f%z}' data.durationSeconds = (stopTime-startTime).total_seconds() data.delayInStartProcessingFromMessage = (startTime - messageInsertionTime).total_seconds()
class AzureQueue(object): def __init__(self, queue_name): self.conn = QueueService(account_name=os.getenv('AZURE_ACCOUNT_NAME'), account_key=os.getenv('AZURE_ACCOUNT_KEY')) self.queue_name = queue_name self.conn.create_queue(queue_name) self.conn.encode_function = QueueMessageFormat.binary_base64encode self.conn.decode_function = QueueMessageFormat.binary_base64decode def enqueue(self, func, *args, **kwargs): task = SimpleTask(func, *args, **kwargs) serialized_task = pickle.dumps(task, protocol=pickle.HIGHEST_PROTOCOL) self.conn.put_message(self.queue_name, serialized_task) return task.id def dequeue(self): messages = self.conn.get_messages(self.queue_name) if len(messages) == 1: serialized_task = messages[0] task = pickle.loads(serialized_task.content) self.conn.delete_message(self.queue_name, serialized_task.id, serialized_task.pop_receipt) return task def get_length(self): metadata = self.conn.get_queue_metadata(self.queue_name) return metadata.approximate_message_count
def main(): storage_account_name = str(sys.argv[1]) queue_name = str(sys.argv[2]) destination_folder = str(sys.argv[3]) if not storage_account_name or not queue_name or not destination_folder: print("usage: get_message.py storage_account_name queue_name destination_folder") sys.exit(1) base64_regex = re.compile("^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{3}=|[A-Za-z0-9+/]{2}==)?$") key_file = "{0}/{1}/acc_key.txt".format(destination_folder, storage_account_name) try : with open(key_file, "r") as f: account_key = f.read() except IOError: print("file not found or empty ", key_file) queue_service = QueueService(storage_account_name, account_key) metadata = queue_service.get_queue_metadata(queue_name) approximate_message_count = metadata.approximate_message_count print('fetching approximately {0} messages'.format(approximate_message_count)) messages = queue_service.get_messages(queue_name) count = len(messages) total = 0 while count > 0: for message in messages: if base64_regex.match(message.content): msg = base64.b64decode(message.content) # Received message is encoded in base 64 data = json.loads(msg.decode('utf8')) else: msg = message.content data = json.loads(msg) filename, folder = get_file_name(storage_account_name, queue_name, destination_folder) if not os.path.exists(folder): os.makedirs(folder) with open(r'{0}/{1}'.format(folder,filename), 'w') as f: json.dump(data, f, indent=2) queue_service.delete_message(queue_name, message.id, message.pop_receipt) count -= 1 total += 1 print('successfully processed {0} messages of ~{1} from {2} queue {3}'.format(total,approximate_message_count,storage_account_name,queue_name)) messages = queue_service.get_messages(queue_name) count = len(messages)
def get_queue_details(creds, resource_group_name, account_name, queue_name): keys = _get_storage_account_keys(creds, resource_group_name, account_name) queue_service = QueueService(account_name, keys.key1) model = StorageAccountQueueDetails() model.queue_name = queue_name model.metadata = queue_service.get_queue_metadata(queue_name) count = int(model.metadata.get('x-ms-approximate-messages-count', '0')) model.messages = queue_service.peek_messages(queue_name, count) if count else [] for msg in model.messages: try: msg.decoded_text = b64decode(msg.message_text).decode() except: msg.decoded_text = None return model
def get_queue_details(creds, resource_group_name, account_name, queue_name): keys = _get_storage_account_keys(creds, resource_group_name, account_name) queue_service = QueueService(account_name, keys.key1) model = StorageAccountQueueDetails() model.queue_name = queue_name model.metadata = queue_service.get_queue_metadata(queue_name) count = int(model.metadata.get("x-ms-approximate-messages-count", "0")) model.messages = queue_service.peek_messages(queue_name, count) if count else [] for msg in model.messages: try: msg.decoded_text = b64decode(msg.message_text).decode() except: msg.decoded_text = None return model
class AzureQueue(object): def __init__(self, account_name, account_key, queue_name): self.queue_name = queue_name self.queue_service = QueueService(account_name=account_name, account_key=account_key) self.queue_service.create_queue(self.queue_name) def put_message_into_queue(self, content) -> QueueMessage: """ Publishes a message with `content` :param content: The queue message :returns: A QueueMessage that has the message as well as metadata :rtype: QueueMessage """ return self.queue_service.put_message(self.queue_name, content) def get_messages(self) -> list: """ Retrieves all of the messages that have been published into queue :param content: The queue message :returns: List of Queue messages :rtype: list """ return self.queue_service.get_messages(self.queue_name) def delete_message_from_queue(self, message_id, pop_receipt): self.queue_service.delete_message(self.queue_name, message_id, pop_receipt) def get_message_count(self): queue_metadata = self.queue_service.get_queue_metadata(self.queue_name) return queue_metadata.approximate_message_count def delete(self): return self.queue_service.delete_queue(self.queue_name) def empty(self): messages = queue_service.get_messages(self.queue_name, num_messages=BATCH_NUMBER, visibility_timeout=TIMEOUT_IN_SECONDS) for message in messages: self.queue_service.delete_message(self.queue_name, message.id, message.pop_receipt)
def account_sas(self): queue_name = self._create_queue() metadata = {'val1': 'foo', 'val2': 'blah'} self.service.set_queue_metadata(queue_name, metadata=metadata) # Access to read operations on the queues themselves # Expires in an hour token = self.service.generate_account_shared_access_signature( ResourceTypes.CONTAINER, AccountPermissions.READ, datetime.utcnow() + timedelta(hours=1), ) # Create a service and use the SAS sas_service = QueueService( account_name=self.account.account_name, sas_token=token, ) metadata = sas_service.get_queue_metadata(queue_name) # metadata={'val1': 'foo', 'val2': 'blah'} self.service.delete_queue(queue_name)
azureQueueAccountName = "" azureQueueKey = "" azureQueueAnalysisRecognizedItems = "recognizeditems-processing" azureQueueAnalysisResults = "analysis-results" azureQueueImageResults = "image-processing" queue_service = QueueService(account_name=azureQueueAccountName, account_key=azureQueueKey) while True: #create queue if doesnt exist if not queue_service.exists(azureQueueAnalysisRecognizedItems): queue_service.create_queue(azureQueueAnalysisRecognizedItems) if not queue_service.exists(azureQueueImageResults): queue_service.create_queue(azureQueueImageResults) #get queue count metadata = queue_service.get_queue_metadata( azureQueueAnalysisResults) queue_count = metadata.approximate_message_count if queue_count > 0: read_next_in_queue() else: logger.info("time.sleep(3000)") time.sleep(3000) except Exception as e: logger.error("Exception in main {}".format(e))
import os from azure.storage.queue import QueueService import time # Grab environment variables. AZURE_STORAGE_ACCT = os.environ['AZURE_STORAGE_ACCT'] AZURE_QUEUE = os.environ['AZURE_QUEUE'] AZURE_QUEUE_KEY = os.environ['AZURE_QUEUE_KEY'] # Build queue object queue_service = QueueService(account_name=AZURE_STORAGE_ACCT, account_key=AZURE_QUEUE_KEY) while True: # Get queue count metadata = queue_service.get_queue_metadata(AZURE_QUEUE) queue_length = metadata.approximate_message_count print(queue_length) messages = queue_service.get_messages(AZURE_QUEUE, num_messages=32) for message in messages: queue_service.delete_message(AZURE_QUEUE, message.id, message.pop_receipt) print("Message deleted")
class AzureService(object): VISIBILITY_TIMEOUT = 5*60 def __init__(self, connection_string, container_name, queue_get, queue_push, logger=None): self.ctnname = container_name self.getname = queue_get self.pushname = queue_push self.qs = QueueService(connection_string=connection_string, protocol='https', # endpoint_suffix='core.windows.net' ) self.bs = BlockBlobService(connection_string=connection_string) self.qs.create_queue(self.getname, timeout=1) self.qs.create_queue(self.pushname, timeout=1) self.bs.create_container(self.ctnname, timeout=1) if logger: logger.info('Init Azure success') def pushMessage(self, message, qname=None, logger=None): if qname is None: qname = self.pushname try: self.qs.put_message(self.pushname, message) except Exception as e: if logger: logger.exception('ERROR PUSH MESSAGE ') else: print 'ERROR PUSH MESSAGE ' print e def getMessage(self, qname=None, num=1, logger=None): if qname is None: qname = self.getname try: message = self.qs.get_messages(qname, num, visibility_timeout=self.VISIBILITY_TIMEOUT) except Exception as e: if logger: logger.exception('ERROR GET MESSAGE ') else: print 'ERROR GET MESSAGE ' print e return [] return message def getReceiptInfo(self, logger=None): message = self.getMessage(logger=logger) if len(message) > 0: rinfo = ReceiptSerialize.fromjson(message[0].content) return message[0], rinfo else: return None, None def count(self): metadata_get = self.qs.get_queue_metadata(self.getname) metadata_push = self.qs.get_queue_metadata(self.pushname) generator = self.bs.list_blobs(self.ctnname) bc = 0 for blob in generator: bc += 1 return {'get_count' : metadata_get.approximate_message_count, 'push_count': metadata_push.approximate_message_count, 'blob_count': bc } def uploadFolder(self, folderpath, logger): for filename in os.listdir(folderpath): if len(filename) > 4: suffix = filename[-4:].upper() else: continue if '.JPG' == suffix or 'JPEG' == suffix: receipt_metadata = ReceiptSerialize() receipt_metadata.receiptBlobName = unicode(filename, 'utf-8') self.qs.put_message(self.getname, b64encode(receipt_metadata.toString()).decode('utf-8')) self.bs.create_blob_from_path(self.ctnname, receipt_metadata.receiptBlobName, os.path.join(folderpath, filename), max_connections=2, timeout=None) logger.info('upload %s', filename) def getImage(self, imgname, logger=None): localpath= os.path.join(args.download_dir, imgname) try: self.bs.get_blob_to_path(self.ctnname, imgname, localpath) except AzureMissingResourceHttpError as e: if logger: logger.error('Blob named ' + imgname + ' doesnot exist.' , exc_info=True) else: print 'Blob named ' + imgname + ' doesnot exist.' print e return '' except Exception as e: if logger: logger.error('Exception while getting blob.', exc_info=True) else: print 'Exception while getting blob.' print e return None return localpath def deleteMessage(self, message, qname=None, logger=None): if qname is None: qname = self.getname try: self.qs.delete_message(qname, message.id, message.pop_receipt) except Exception as e: if logger: logger.exception('ERROR DELETE MESSAGE ') else: print 'ERROR DELETE MESSAGE ' print e def deleteImage(self, imgname, logger=None): try: self.bs.delete_blob(self.ctnname, imgname) except Exception as e: if logger: logger.exception('ERROR DELETE IMAGE ') else: print 'ERROR DELETE IMAGE ' print e def cleanUp(self): count = 0 print('deleted: ') while True: messages = self.qs.get_messages(self.getname) for message in messages: count += 1 self.qs.delete_message(self.getname, message.id, message.pop_receipt) if len(messages) == 0: break print(str(count) + ' from queue-get') count = 0 while True: messages = self.qs.get_messages(self.pushname) for message in messages: count += 1 self.qs.delete_message(self.pushname, message.id, message.pop_receipt) if len(messages) == 0: break print(str(count) + ' from queue-push') count = 0 generator = self.bs.list_blobs(self.ctnname) for blob in generator: count += 1 self.bs.delete_blob(self.ctnname, blob.name) print(str(count) + ' from container')
class Queue: def __init__(self, account_name, account_key, queue_name="logqueue"): """Initialiaze a queue. The type is set by the 'ACS_LOGGING_QUEUE_TYPE' environment variable. If it is set to 'AzureStorageQueue' then values must be provided for 'account_name' and 'account_key' which are values associated with the Azure Storage account. 'queue_name' is optional and defaults to 'logqueue'. """ self.log = Log() self.queue_type = config.ACS_LOGGING_QUEUE_TYPE self.queue_name = queue_name # self.log.debug("Queue type: " + self.queue_type + " / " + self.queue_name) if self.queue_type == "AzureStorageQueue": self.createAzureQueues(account_name, account_key) elif self.queue_type == "LocalFile": self.file_queue = open(config.UNPROCESSED_LOG_FILE, 'w+') else: self.log.error("Unknown queue type: " + queue_type) def getName(self): return self.queue_name def createAzureQueues(self, account_name, account_key): """ Create a queue for unprocessed log messages. Entries in the queue will be dequeued, processed and deleted upon success. """ self.queue_service = QueueService(account_name, account_key) self.queue_service.create_queue(self.queue_name) def close(self): """Perform any necessary clearnup on the queue at the end of a run. """ if self.queue_type == "AzureStorageQueue": pass elif self.queue_type == "LocalFile": self.file_queue.close() else: self.log.error("Unknown queue type: " + queue_type) def enqueue(self, msg): if self.queue_type == "LocalFile": file_queue.write(msg + '\n') elif self.queue_type == "AzureStorageQueue": self.queue_service.put_message(self.queue_name, msg) else: self.log.error("We don't know how to handle queues of type " + self.queue_type) self.log.debug(msg) def dequeue(self): messages = [] if self.queue_type == "LocalFile": with open(config.UNPROCESSED_LOG_FILE, 'r') as f: messages = f.readlines()[1] elif self.queue_type == "AzureStorageQueue": messages = self.queue_service.get_messages(self.queue_name) return messages def delete(self, message): self.queue_service.delete_message(self.queue_name, message.message_id, message.pop_receipt) # with open(config.PROCESSED_LOG_FILE, 'a') as processed: # processed.write(log) # os.remove(config.UNPROCESSED_LOG_FILE) def delete_queue(self, queue_name): self.queue_service.delete_queue(queue_name, False) def getLength(self): """ Get the approximate length of the queue """ queue_metadata = self.queue_service.get_queue_metadata(self.queue_name) count = queue_metadata['x-ms-approximate-messages-count'] return int(count) def peek_messages(self, num_messages): """ Peek at the top messages in the queue. This method does not remove the messages from the queue. """ return self.queue_service.peek_messages(self.queue_name, num_messages)
from azure.storage.queue import QueueService import sys import json import datetime import socket import pprint import logger log = logger.setup_custom_logger('consumer') try: with open('config.json') as config_file: config = json.load(config_file) except FileNotFoundError: print("No config.json set") exit(1) print("Getting meta from: " + config['queueName']) queue_service = QueueService(account_name=config['accountName'], account_key=config['accountKey']) metadata = queue_service.get_queue_metadata(config['queueName']) log.info(metadata.approximate_message_count)
except Exception: logger.error("Exception in final_frame") logger.warn(link) if __name__ == '__main__': loggerName = "ResultsLinksProcess" logger = logging.getLogger(loggerName) azureQueueAccountName = "traxwebjobs" azureQueueKey = "" azureQueueImageProcess = "image-processing" queue_service = QueueService(account_name=azureQueueAccountName, account_key=azureQueueKey) try: while True: #get queue count metadata = queue_service.get_queue_metadata(azureQueueImageProcess) queue_count = metadata.approximate_message_count if queue_count > 0: read_next_in_queue() else: logger.info("time.sleep(1)") time.sleep(1) except Exception as e: init_logger() logger.error("Exception in main {}".format(e))
config.read('storage.conf') accName = config.get('storage_account', 'accName') accKey = config.get('storage_account', 'accKey') queueName = config.get('storage_account', 'queueName') # sudo pip install blessings from azure.storage.queue import QueueService, QueueMessageFormat from blessings import Terminal import sys import time queue_service = QueueService(account_name=accName, account_key=accKey) queue_service.decode_function = QueueMessageFormat.text_base64decode t = Terminal() print(t.green('Connected to Azure Storage Queue ' + queueName + '...')) # Get approximate number of messages in queue queue_metadata = queue_service.get_queue_metadata(queueName) count = queue_metadata.approximate_message_count print('Approximate number of messages in queue: ', count, '\n') while True: messages = queue_service.get_messages(queueName) if messages: # Get the next message for message in messages: print(t.bold_yellow(message.content)) queue_service.delete_message(queueName, message.id, message.pop_receipt) print(t.blue('-' * 40 + '\n')) time.sleep(4)
queue_service.put_message('pizzaqueue', 'Pepperoni pizza ordered.') queue_service.put_message('pizzaqueue', 'Hawiian pizza ordered.') queue_service.put_message('pizzaqueue', 'Pepperoni pizza ordered.') queue_service.put_message('pizzaqueue', 'Pepperoni pizza ordered.') time.sleep(1) ### # Use the Azure Storage Storage SDK for Python to count how many messages are in the Queue ### print( '\nLet\'s see how many orders we have to start cooking! Here, we simply examine how many messages are sitting the Queue. ' ) input('Press Enter to continue...') metadata = queue_service.get_queue_metadata('pizzaqueue') print(('Number of messages in the queue: ' + str(metadata.approximate_message_count))) time.sleep(1) ### # Use the Azure Storage Storage SDK for Python to read each message from the Queue ### print( '\nWith some messages in our Azure Storage Queue, let\'s read the first message in the Queue to signal we start to process that customer\'s order.' ) input('Press Enter to continue...') # When you get each message, they become hidden from other parts of the applications being able to see it. # Once you have successfully processed the message, you then delete the message from the Queue.
from azure.storage.queue import QueueService # Grab environment variables. azurestoracct = os.environ['azurestoracct'] azurequeue = os.environ['azurequeue'] azurequeuekey = os.environ['azurequeuekey'] + "==;" docker_image = os.environ['docker_image'] docker_service = os.environ['docker_service'] queue_length = os.environ['queuelength'] while True: # Set up azure queue / get count of messages on queue (count). queue_service = QueueService(account_name=azurestoracct, account_key=azurequeuekey) metadata = queue_service.get_queue_metadata(azurequeue) count = metadata.approximate_message_count if "docker" in os.environ: # Get Docker address from environment variable docker = os.environ['docker'] # Get Marathon task from API if (requests.get(docker + "/services/" + docker_service)): service = json.loads( (requests.get(docker + "/services/" + docker_service)).text) service_version = service['Version']['Index'] replica_count = service['Spec']['Mode']['Replicated']['Replicas'] service_id = service['ID']
"\n\nThen, wait 6 seconds, dequeue the message and print its content.\n" ) sleep(6) else: print("\n\nSuccessfully deleted message!\n") finally: messages = queue_service.get_messages('umairqueue', num_messages=1) for message in messages: print(message.content) queue_service.delete_message('umairqueue', message.id, message.pop_receipt) input( "\n\nLeave one message in the queue and do not delete the queue right away.\n Press any key to contine..." ) metadata = queue_service.get_queue_metadata('umairqueue') while (metadata.approximate_message_count > 1): messages = queue_service.get_messages('umairqueue', num_messages=1) for message in messages: print("Deleting message: " + message.content + '\n\n') queue_service.delete_message('umairqueue', message.id, message.pop_receipt) metadata = queue_service.get_queue_metadata('umairqueue') message = queue_service.peek_messages('umairqueue') print("Last message left in the queue: " + message[0].content + "\n\n") input("Deleting queue, press any key to proceed?\n") queue_service.delete_queue('umairqueue')
from azure.storage.queue import QueueService queue_service = QueueService( account_name='intellimannstorage', account_key= 'agA19rC5bc9JjbcCb8vyya/OqXCxrP4slPOG8L+dgVmdmKk3HRbgFu+f2W8eZDhc/uiMhadKIZseL9z1nHFwow==' ) messages = queue_service.get_messages('myqueue') metadata = queue_service.get_queue_metadata('myqueue') count = metadata.approximate_message_count print(count) for message in messages: print(__dir__(message)) print(message.content) print(type(message.content)) queue_service.delete_message('myqueue', message.id, message.pop_receipt)
class Queue: def __init__(self, account_name, account_key, queue_name="logqueue"): """Initialiaze a queue. The type is set by the 'ACS_LOGGING_QUEUE_TYPE' environment variable. If it is set to 'AzureStorageQueue' then values must be provided for 'account_name' and 'account_key' which are values associated with the Azure Storage account. 'queue_name' is optional and defaults to 'logqueue'. """ self.log = Log() self.queue_type = config.ACS_LOGGING_QUEUE_TYPE self.queue_name = queue_name # self.log.debug("Queue type: " + self.queue_type + " / " + self.queue_name) if self.queue_type == "AzureStorageQueue": self.createAzureQueues(account_name, account_key) elif self.queue_type == "LocalFile": self.file_queue = open(config.UNPROCESSED_LOG_FILE, "w+") else: self.log.error("Unknown queue type: " + queue_type) def createAzureQueues(self, account_name, account_key): """ Create a queue for unprocessed log messages. Entries in the queue will be dequeued, processed and deleted upon success. """ self.queue_service = QueueService(account_name, account_key) self.queue_service.create_queue(self.queue_name) def close(self): """Perform any necessary clearnup on the queue at the end of a run. """ if self.queue_type == "AzureStorageQueue": pass elif self.queue_type == "LocalFile": self.file_queue.close() else: self.log.error("Unknown queue type: " + queue_type) def enqueue(self, msg, level="INFO"): msg = level + " - " + msg if self.queue_type == "LocalFile": file_queue.write(msg + "\n") elif self.queue_type == "AzureStorageQueue": self.queue_service.put_message(self.queue_name, msg) self.log.debug(msg) def dequeue(self): messages = [] if self.queue_type == "LocalFile": with open(config.UNPROCESSED_LOG_FILE, "r") as f: messages = f.readlines()[1] elif self.queue_type == "AzureStorageQueue": messages = self.queue_service.get_messages(self.queue_name) return messages def delete(self, message): self.queue_service.delete_message(self.queue_name, message.message_id, message.pop_receipt) # with open(config.PROCESSED_LOG_FILE, 'a') as processed: # processed.write(log) # os.remove(config.UNPROCESSED_LOG_FILE) def delete_queue(self, queue_name): self.queue_service.delete_queue(queue_name, False) def getLength(self): """ Get the approximate length of the queue """ queue_metadata = self.queue_service.get_queue_metadata(self.queue_name) count = queue_metadata["x-ms-approximate-messages-count"] return count
# Queue reader - Simple script to read and print entries in the queue # Pedro Perez - 2015 # # ----------------------------------------------------------------------------- import sys import os import subprocess from azure.storage.queue import QueueService account_name = "mcdockerqueue" account_key = sys.argv[1] queuename = "servers" queue_service = QueueService(account_name, account_key) queue_metadata = queue_service.get_queue_metadata(queuename) count = queue_metadata['x-ms-approximate-messages-count'] print "There are %s messages in the queue" % count if count > 0: messages = queue_service.get_messages(queuename) for message in messages: print(message.message_text) # Remove message from the queue queue_service.delete_message(queuename, message.message_id, message.pop_receipt) else: print "There are no messages to process"
logger.warn(link) if __name__ == '__main__': loggerName = "recognizeditems" logger = logging.getLogger(loggerName) azureQueueAccountName = "" azureQueueKey = "" azureQueueRecognizedItems = "recognizeditems-processing" queue_service = QueueService(account_name=azureQueueAccountName, account_key=azureQueueKey) try: while True: #get queue count metadata = queue_service.get_queue_metadata(azureQueueRecognizedItems) queue_count = metadata.approximate_message_count if queue_count > 0: read_next_in_queue() else: logger.info("time.sleep(3000)") time.sleep(3000) except Exception as e: init_logger() logger.error("Exception in main {}" .format(e))