def processMessage(data: DocumentData, startTime: datetime):
    startTime = datetime.utcnow()
    result = doMath()
    result = 0

    args = sys.argv
   
    data.start = f'{startTime:%Y-%m-%d %H:%M:%S.%f%z}' 
    data.host = args[2]
    data.port = args[4]
    data.worker = args[6]
    data.request = args[8]
    data.plataform = sys.platform
    data.node = platform.node()
    data.result = result
    data.version = "14" # Identifyier to filter logs
  
    # Calculating queue size
    storageKey = os.environ['StorageKey']
    queue_service = QueueService(account_name='funcpyqueue2storage', account_key=storageKey)
    metadata = queue_service.get_queue_metadata('fibonaccicalculatorqueue')
    data.queuedMessages = metadata.approximate_message_count

    stopTime = datetime.utcnow()
    
    # Calculate times
    messageInsertionTime = datetime.strptime(data.msgInsertionTime, '%Y-%m-%d %H:%M:%S')
    data.stop = f'{stopTime:%Y-%m-%d %H:%M:%S.%f%z}'
    data.durationSeconds = (stopTime-startTime).total_seconds()
    data.delayInStartProcessingFromMessage = (startTime - messageInsertionTime).total_seconds()
    
Beispiel #2
0
class AzureQueue(object):
    def __init__(self, queue_name):
        self.conn = QueueService(account_name=os.getenv('AZURE_ACCOUNT_NAME'),
                                 account_key=os.getenv('AZURE_ACCOUNT_KEY'))
        self.queue_name = queue_name
        self.conn.create_queue(queue_name)
        self.conn.encode_function = QueueMessageFormat.binary_base64encode
        self.conn.decode_function = QueueMessageFormat.binary_base64decode

    def enqueue(self, func, *args, **kwargs):
        task = SimpleTask(func, *args, **kwargs)
        serialized_task = pickle.dumps(task, protocol=pickle.HIGHEST_PROTOCOL)
        self.conn.put_message(self.queue_name, serialized_task)
        return task.id

    def dequeue(self):
        messages = self.conn.get_messages(self.queue_name)
        if len(messages) == 1:
            serialized_task = messages[0]
            task = pickle.loads(serialized_task.content)
            self.conn.delete_message(self.queue_name, serialized_task.id,
                                     serialized_task.pop_receipt)
            return task

    def get_length(self):
        metadata = self.conn.get_queue_metadata(self.queue_name)
        return metadata.approximate_message_count
Beispiel #3
0
def main():    
    storage_account_name = str(sys.argv[1])
    queue_name = str(sys.argv[2])
    destination_folder = str(sys.argv[3])
    
    if not storage_account_name or not queue_name or not destination_folder:
        print("usage: get_message.py storage_account_name queue_name destination_folder")
        sys.exit(1)
    
    base64_regex = re.compile("^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{3}=|[A-Za-z0-9+/]{2}==)?$")  
    
    key_file = "{0}/{1}/acc_key.txt".format(destination_folder, storage_account_name)
    try :
        with open(key_file, "r") as f:
            account_key = f.read()
    except IOError:
        print("file not found or empty ", key_file)
    
    queue_service = QueueService(storage_account_name, account_key)

    metadata = queue_service.get_queue_metadata(queue_name)
    approximate_message_count = metadata.approximate_message_count
    print('fetching approximately {0} messages'.format(approximate_message_count))
    messages = queue_service.get_messages(queue_name)
    count = len(messages)
    total = 0
    while count > 0:
        for message in messages:
            if base64_regex.match(message.content):
                msg = base64.b64decode(message.content)     # Received message is encoded in base 64
                data = json.loads(msg.decode('utf8'))
            else:
                msg = message.content
                data = json.loads(msg)
                
            filename, folder = get_file_name(storage_account_name, queue_name, destination_folder)
            
            if not os.path.exists(folder):
                os.makedirs(folder)
            with open(r'{0}/{1}'.format(folder,filename), 'w') as f:
                json.dump(data, f, indent=2)
            queue_service.delete_message(queue_name, message.id, message.pop_receipt) 
            count -= 1
            total += 1
            print('successfully processed {0} messages of ~{1} from {2} queue {3}'.format(total,approximate_message_count,storage_account_name,queue_name))

        messages = queue_service.get_messages(queue_name)
        count = len(messages)
Beispiel #4
0
def get_queue_details(creds, resource_group_name, account_name, queue_name):
    keys = _get_storage_account_keys(creds, resource_group_name, account_name)
    queue_service = QueueService(account_name, keys.key1)

    model = StorageAccountQueueDetails()
    model.queue_name = queue_name
    model.metadata = queue_service.get_queue_metadata(queue_name)
    count = int(model.metadata.get('x-ms-approximate-messages-count', '0'))
    model.messages = queue_service.peek_messages(queue_name, count) if count else []
    for msg in model.messages:
        try:
            msg.decoded_text = b64decode(msg.message_text).decode()
        except:
            msg.decoded_text = None

    return model
Beispiel #5
0
def get_queue_details(creds, resource_group_name, account_name, queue_name):
    keys = _get_storage_account_keys(creds, resource_group_name, account_name)
    queue_service = QueueService(account_name, keys.key1)

    model = StorageAccountQueueDetails()
    model.queue_name = queue_name
    model.metadata = queue_service.get_queue_metadata(queue_name)
    count = int(model.metadata.get("x-ms-approximate-messages-count", "0"))
    model.messages = queue_service.peek_messages(queue_name, count) if count else []
    for msg in model.messages:
        try:
            msg.decoded_text = b64decode(msg.message_text).decode()
        except:
            msg.decoded_text = None

    return model
Beispiel #6
0
class AzureQueue(object):

  def __init__(self, account_name, account_key, queue_name):
    self.queue_name = queue_name
    self.queue_service = QueueService(account_name=account_name, account_key=account_key)
    self.queue_service.create_queue(self.queue_name)

  def put_message_into_queue(self, content) -> QueueMessage:
    """
    Publishes a message with `content`
    
    :param content: The queue message 

    :returns: A QueueMessage that has the message as well as metadata 
    :rtype: QueueMessage 
    """
    return self.queue_service.put_message(self.queue_name, content)

  def get_messages(self) -> list:
    """
    Retrieves all of the messages that have been published into queue 
    
    :param content: The queue message 

    :returns: List of Queue messages
    :rtype: list 
    """

    return self.queue_service.get_messages(self.queue_name)

  def delete_message_from_queue(self, message_id, pop_receipt):
    self.queue_service.delete_message(self.queue_name, message_id, pop_receipt)

  def get_message_count(self):
    queue_metadata = self.queue_service.get_queue_metadata(self.queue_name)
    return queue_metadata.approximate_message_count

  def delete(self):
    return self.queue_service.delete_queue(self.queue_name)

  def empty(self):
    messages = queue_service.get_messages(self.queue_name, 
                                          num_messages=BATCH_NUMBER, 
                                          visibility_timeout=TIMEOUT_IN_SECONDS) 
    for message in messages:
      self.queue_service.delete_message(self.queue_name, message.id, message.pop_receipt)
    def account_sas(self):
        queue_name = self._create_queue()
        metadata = {'val1': 'foo', 'val2': 'blah'}
        self.service.set_queue_metadata(queue_name, metadata=metadata)

        # Access to read operations on the queues themselves
        # Expires in an hour
        token = self.service.generate_account_shared_access_signature(
            ResourceTypes.CONTAINER,
            AccountPermissions.READ,
            datetime.utcnow() + timedelta(hours=1),
        )

        # Create a service and use the SAS
        sas_service = QueueService(
            account_name=self.account.account_name,
            sas_token=token,
        )
        metadata = sas_service.get_queue_metadata(queue_name) # metadata={'val1': 'foo', 'val2': 'blah'}

        self.service.delete_queue(queue_name)
        azureQueueAccountName = ""
        azureQueueKey = ""
        azureQueueAnalysisRecognizedItems = "recognizeditems-processing"
        azureQueueAnalysisResults = "analysis-results"
        azureQueueImageResults = "image-processing"

        queue_service = QueueService(account_name=azureQueueAccountName,
                                     account_key=azureQueueKey)

        while True:
            #create queue if doesnt exist
            if not queue_service.exists(azureQueueAnalysisRecognizedItems):
                queue_service.create_queue(azureQueueAnalysisRecognizedItems)

            if not queue_service.exists(azureQueueImageResults):
                queue_service.create_queue(azureQueueImageResults)

            #get queue count
            metadata = queue_service.get_queue_metadata(
                azureQueueAnalysisResults)
            queue_count = metadata.approximate_message_count

            if queue_count > 0:
                read_next_in_queue()
            else:
                logger.info("time.sleep(3000)")
                time.sleep(3000)

    except Exception as e:
        logger.error("Exception in main {}".format(e))
Beispiel #9
0
import os
from azure.storage.queue import QueueService
import time

# Grab environment variables.
AZURE_STORAGE_ACCT = os.environ['AZURE_STORAGE_ACCT']
AZURE_QUEUE = os.environ['AZURE_QUEUE']
AZURE_QUEUE_KEY = os.environ['AZURE_QUEUE_KEY']

# Build queue object
queue_service = QueueService(account_name=AZURE_STORAGE_ACCT,
                             account_key=AZURE_QUEUE_KEY)

while True:

    # Get queue count
    metadata = queue_service.get_queue_metadata(AZURE_QUEUE)
    queue_length = metadata.approximate_message_count
    print(queue_length)

    messages = queue_service.get_messages(AZURE_QUEUE, num_messages=32)
    for message in messages:
        queue_service.delete_message(AZURE_QUEUE, message.id,
                                     message.pop_receipt)
        print("Message deleted")
class AzureService(object):
    VISIBILITY_TIMEOUT = 5*60

    def __init__(self, connection_string, container_name, queue_get, queue_push, logger=None):
        self.ctnname = container_name
        self.getname = queue_get
        self.pushname = queue_push
        
        self.qs = QueueService(connection_string=connection_string,
                               protocol='https',
#                                endpoint_suffix='core.windows.net'
                                )
        self.bs = BlockBlobService(connection_string=connection_string)
        self.qs.create_queue(self.getname, timeout=1)
        self.qs.create_queue(self.pushname, timeout=1)
        self.bs.create_container(self.ctnname, timeout=1)
        if logger: logger.info('Init Azure success')
    
    def pushMessage(self, message, qname=None, logger=None):
        if qname is None:
            qname = self.pushname
        try:
            self.qs.put_message(self.pushname, message) 
        except Exception as e:
            if logger:
                logger.exception('ERROR PUSH MESSAGE ')
            else:
                print 'ERROR PUSH MESSAGE '
                print e
        
    def getMessage(self, qname=None, num=1, logger=None):
        if qname is None:
            qname = self.getname
        try:
            message = self.qs.get_messages(qname, num, visibility_timeout=self.VISIBILITY_TIMEOUT)
        except Exception as e:
            if logger:
                logger.exception('ERROR GET MESSAGE ')
            else:
                print 'ERROR GET MESSAGE '
                print e
            return []
        return message
    
    def getReceiptInfo(self, logger=None):
        message = self.getMessage(logger=logger)
        if len(message) > 0:
            rinfo = ReceiptSerialize.fromjson(message[0].content)   
            return message[0], rinfo
        else:
            return None, None
        
    def count(self):
        metadata_get = self.qs.get_queue_metadata(self.getname)
        metadata_push = self.qs.get_queue_metadata(self.pushname)
        generator = self.bs.list_blobs(self.ctnname)
        bc = 0
        for blob in generator:
            bc += 1
        return {'get_count' : metadata_get.approximate_message_count, 
                'push_count': metadata_push.approximate_message_count,
                'blob_count': bc
                } 
    
    def uploadFolder(self, folderpath, logger):
        for filename in os.listdir(folderpath):
            if len(filename) > 4:
                suffix = filename[-4:].upper()
            else:
                continue
            if '.JPG' == suffix or 'JPEG' == suffix:
                receipt_metadata = ReceiptSerialize()
                receipt_metadata.receiptBlobName = unicode(filename, 'utf-8')
                self.qs.put_message(self.getname, b64encode(receipt_metadata.toString()).decode('utf-8')) 
                self.bs.create_blob_from_path(self.ctnname, receipt_metadata.receiptBlobName, os.path.join(folderpath, filename), max_connections=2, timeout=None)
                logger.info('upload %s', filename)
    
    def getImage(self, imgname, logger=None):
        localpath= os.path.join(args.download_dir, imgname)
        try:
            self.bs.get_blob_to_path(self.ctnname, imgname, localpath)
        except AzureMissingResourceHttpError as e:
            if logger:
                logger.error('Blob named ' + imgname + ' doesnot exist.' , exc_info=True)
            else:
                print 'Blob named ' + imgname + ' doesnot exist.' 
                print e            
            return ''
        except Exception as e:
            if logger:
                logger.error('Exception while getting blob.', exc_info=True)
            else:
                print 'Exception while getting blob.' 
                print e            
            return None
        return localpath
    
    def deleteMessage(self, message, qname=None, logger=None): 
        if qname is None:
            qname = self.getname
        try:
            self.qs.delete_message(qname, message.id, message.pop_receipt)
        except Exception as e:
            if logger:
                logger.exception('ERROR DELETE MESSAGE ')
            else:
                print 'ERROR DELETE MESSAGE '
                print e
                
    def deleteImage(self, imgname, logger=None):
        try: 
            self.bs.delete_blob(self.ctnname, imgname)
        except Exception as e:
            if logger:
                logger.exception('ERROR DELETE IMAGE ')
            else:
                print 'ERROR DELETE IMAGE '
                print e
        
    def cleanUp(self):
        count = 0
        print('deleted: ')
        while True:
            messages = self.qs.get_messages(self.getname)
            for message in messages:
                count += 1
                self.qs.delete_message(self.getname, message.id, message.pop_receipt)
            if len(messages) == 0: break
        print(str(count) + ' from queue-get')
        count = 0
        while True:
            messages = self.qs.get_messages(self.pushname)
            for message in messages:
                count += 1
                self.qs.delete_message(self.pushname, message.id, message.pop_receipt)
            if len(messages) == 0: break
        print(str(count) + ' from queue-push') 
        count = 0
        generator = self.bs.list_blobs(self.ctnname)
        for blob in generator:
            count += 1     
            self.bs.delete_blob(self.ctnname, blob.name)
        print(str(count) + ' from container') 
class Queue:
    def __init__(self, account_name, account_key, queue_name="logqueue"):
        """Initialiaze a queue. The type is set by the
        'ACS_LOGGING_QUEUE_TYPE' environment variable. If it is set to
        'AzureStorageQueue' then values must be provided for
        'account_name' and 'account_key' which are values associated
        with the Azure Storage account. 'queue_name' is optional and
        defaults to 'logqueue'.

        """
        self.log = Log()
        self.queue_type = config.ACS_LOGGING_QUEUE_TYPE
        self.queue_name = queue_name
        # self.log.debug("Queue type: " + self.queue_type + " / " + self.queue_name)

        if self.queue_type == "AzureStorageQueue":
            self.createAzureQueues(account_name, account_key)
        elif self.queue_type == "LocalFile":
            self.file_queue = open(config.UNPROCESSED_LOG_FILE, 'w+')
        else:
            self.log.error("Unknown queue type: " + queue_type)

    def getName(self):
        return self.queue_name
        
    def createAzureQueues(self, account_name, account_key):
        """
        Create a queue for unprocessed log messages. Entries in the queue
        will be dequeued, processed and deleted upon success.
        """
        self.queue_service = QueueService(account_name, account_key)
        self.queue_service.create_queue(self.queue_name)

    def close(self):
        """Perform any necessary clearnup on the queue
           at the end of a run.
        """
        if self.queue_type == "AzureStorageQueue":
            pass
        elif self.queue_type == "LocalFile":
            self.file_queue.close()
        else:
            self.log.error("Unknown queue type: " + queue_type)

    def enqueue(self, msg):
        if self.queue_type == "LocalFile":
            file_queue.write(msg + '\n')
        elif self.queue_type == "AzureStorageQueue":
            self.queue_service.put_message(self.queue_name, msg)
        else:
            self.log.error("We don't know how to handle queues of type " + self.queue_type)
        self.log.debug(msg)

    def dequeue(self):
        messages = []
        if self.queue_type == "LocalFile":
            with open(config.UNPROCESSED_LOG_FILE, 'r') as f:
                messages = f.readlines()[1]
        elif self.queue_type == "AzureStorageQueue":
            messages = self.queue_service.get_messages(self.queue_name)
        return messages

    def delete(self, message):
        self.queue_service.delete_message(self.queue_name, message.message_id, message.pop_receipt)
        #  with open(config.PROCESSED_LOG_FILE, 'a') as processed:
        #    processed.write(log)
        #  os.remove(config.UNPROCESSED_LOG_FILE)

    def delete_queue(self, queue_name):
        self.queue_service.delete_queue(queue_name, False)

    def getLength(self):
        """
        Get the approximate length of the queue
        """
        queue_metadata = self.queue_service.get_queue_metadata(self.queue_name)
        count = queue_metadata['x-ms-approximate-messages-count']
        return int(count)

    def peek_messages(self, num_messages):
        """
        Peek at the top messages in the queue. This method does not remove the
        messages from the queue.
        """
        return self.queue_service.peek_messages(self.queue_name, num_messages)
from azure.storage.queue import QueueService
import sys
import json
import datetime
import socket
import pprint

import logger

log = logger.setup_custom_logger('consumer')

try:
    with open('config.json') as config_file:
        config = json.load(config_file)
except FileNotFoundError:
    print("No config.json set")
    exit(1)

print("Getting meta from: " + config['queueName'])

queue_service = QueueService(account_name=config['accountName'],
                             account_key=config['accountKey'])

metadata = queue_service.get_queue_metadata(config['queueName'])

log.info(metadata.approximate_message_count)
    except Exception:
        logger.error("Exception in final_frame")
        logger.warn(link)


if __name__ == '__main__':
    loggerName = "ResultsLinksProcess"
    logger = logging.getLogger(loggerName)

    azureQueueAccountName = "traxwebjobs"
    azureQueueKey = ""
    azureQueueImageProcess = "image-processing"
    queue_service = QueueService(account_name=azureQueueAccountName,
                                 account_key=azureQueueKey)
    try:

        while True:
            #get queue count
            metadata = queue_service.get_queue_metadata(azureQueueImageProcess)
            queue_count = metadata.approximate_message_count

            if queue_count > 0:
                read_next_in_queue()
            else:
                logger.info("time.sleep(1)")
                time.sleep(1)

    except Exception as e:
        init_logger()
        logger.error("Exception in main {}".format(e))
Beispiel #14
0
config.read('storage.conf')
accName = config.get('storage_account', 'accName')
accKey = config.get('storage_account', 'accKey')
queueName = config.get('storage_account', 'queueName')

# sudo pip install blessings
from azure.storage.queue import QueueService, QueueMessageFormat
from blessings import Terminal
import sys
import time

queue_service = QueueService(account_name=accName, account_key=accKey)
queue_service.decode_function = QueueMessageFormat.text_base64decode

t = Terminal()
print(t.green('Connected to Azure Storage Queue ' + queueName + '...'))
# Get approximate number of messages in queue
queue_metadata = queue_service.get_queue_metadata(queueName)
count = queue_metadata.approximate_message_count
print('Approximate number of messages in queue: ', count, '\n')

while True:
    messages = queue_service.get_messages(queueName)
    if messages:
        # Get the next message
        for message in messages:
            print(t.bold_yellow(message.content))
            queue_service.delete_message(queueName, message.id, message.pop_receipt)
            print(t.blue('-' * 40 + '\n'))
        time.sleep(4)
Beispiel #15
0
queue_service.put_message('pizzaqueue', 'Pepperoni pizza ordered.')
queue_service.put_message('pizzaqueue', 'Hawiian pizza ordered.')
queue_service.put_message('pizzaqueue', 'Pepperoni pizza ordered.')
queue_service.put_message('pizzaqueue', 'Pepperoni pizza ordered.')

time.sleep(1)

###
# Use the Azure Storage Storage SDK for Python to count how many messages are in the Queue
###
print(
    '\nLet\'s see how many orders we have to start cooking! Here, we simply examine how many messages are sitting the Queue. '
)
input('Press Enter to continue...')

metadata = queue_service.get_queue_metadata('pizzaqueue')
print(('Number of messages in the queue: ' +
       str(metadata.approximate_message_count)))

time.sleep(1)

###
# Use the Azure Storage Storage SDK for Python to read each message from the Queue
###
print(
    '\nWith some messages in our Azure Storage Queue, let\'s read the first message in the Queue to signal we start to process that customer\'s order.'
)
input('Press Enter to continue...')

# When you get each message, they become hidden from other parts of the applications being able to see it.
# Once you have successfully processed the message, you then delete the message from the Queue.
from azure.storage.queue import QueueService

# Grab environment variables.
azurestoracct = os.environ['azurestoracct']
azurequeue = os.environ['azurequeue']
azurequeuekey = os.environ['azurequeuekey'] + "==;"
docker_image = os.environ['docker_image']
docker_service = os.environ['docker_service']
queue_length = os.environ['queuelength']

while True:

    # Set up azure queue / get count of messages on queue (count).
    queue_service = QueueService(account_name=azurestoracct,
                                 account_key=azurequeuekey)
    metadata = queue_service.get_queue_metadata(azurequeue)
    count = metadata.approximate_message_count

    if "docker" in os.environ:

        # Get Docker address from environment variable
        docker = os.environ['docker']

        # Get Marathon task from API
        if (requests.get(docker + "/services/" + docker_service)):
            service = json.loads(
                (requests.get(docker + "/services/" + docker_service)).text)
            service_version = service['Version']['Index']
            replica_count = service['Spec']['Mode']['Replicated']['Replicas']
            service_id = service['ID']
Beispiel #17
0
            "\n\nThen, wait 6 seconds, dequeue the message and print its content.\n"
        )
        sleep(6)
    else:
        print("\n\nSuccessfully deleted message!\n")
    finally:
        messages = queue_service.get_messages('umairqueue', num_messages=1)
        for message in messages:
            print(message.content)
            queue_service.delete_message('umairqueue', message.id,
                                         message.pop_receipt)

input(
    "\n\nLeave one message in the queue and do not delete the queue right away.\n Press any key to contine..."
)
metadata = queue_service.get_queue_metadata('umairqueue')
while (metadata.approximate_message_count > 1):
    messages = queue_service.get_messages('umairqueue', num_messages=1)
    for message in messages:
        print("Deleting message: " + message.content + '\n\n')
        queue_service.delete_message('umairqueue', message.id,
                                     message.pop_receipt)
    metadata = queue_service.get_queue_metadata('umairqueue')

message = queue_service.peek_messages('umairqueue')
print("Last message left in the queue: " + message[0].content + "\n\n")

input("Deleting queue, press any key to proceed?\n")

queue_service.delete_queue('umairqueue')
Beispiel #18
0
from azure.storage.queue import QueueService

queue_service = QueueService(
    account_name='intellimannstorage',
    account_key=
    'agA19rC5bc9JjbcCb8vyya/OqXCxrP4slPOG8L+dgVmdmKk3HRbgFu+f2W8eZDhc/uiMhadKIZseL9z1nHFwow=='
)

messages = queue_service.get_messages('myqueue')
metadata = queue_service.get_queue_metadata('myqueue')
count = metadata.approximate_message_count
print(count)
for message in messages:
    print(__dir__(message))
    print(message.content)
    print(type(message.content))
    queue_service.delete_message('myqueue', message.id, message.pop_receipt)
class Queue:
    def __init__(self, account_name, account_key, queue_name="logqueue"):
        """Initialiaze a queue. The type is set by the
        'ACS_LOGGING_QUEUE_TYPE' environment variable. If it is set to
        'AzureStorageQueue' then values must be provided for
        'account_name' and 'account_key' which are values associated
        with the Azure Storage account. 'queue_name' is optional and
        defaults to 'logqueue'.

        """
        self.log = Log()
        self.queue_type = config.ACS_LOGGING_QUEUE_TYPE
        self.queue_name = queue_name
        # self.log.debug("Queue type: " + self.queue_type + " / " + self.queue_name)

        if self.queue_type == "AzureStorageQueue":
            self.createAzureQueues(account_name, account_key)
        elif self.queue_type == "LocalFile":
            self.file_queue = open(config.UNPROCESSED_LOG_FILE, "w+")
        else:
            self.log.error("Unknown queue type: " + queue_type)

    def createAzureQueues(self, account_name, account_key):
        """
        Create a queue for unprocessed log messages. Entries in the queue
        will be dequeued, processed and deleted upon success.
        """
        self.queue_service = QueueService(account_name, account_key)
        self.queue_service.create_queue(self.queue_name)

    def close(self):
        """Perform any necessary clearnup on the queue
           at the end of a run.
        """
        if self.queue_type == "AzureStorageQueue":
            pass
        elif self.queue_type == "LocalFile":
            self.file_queue.close()
        else:
            self.log.error("Unknown queue type: " + queue_type)

    def enqueue(self, msg, level="INFO"):
        msg = level + " - " + msg
        if self.queue_type == "LocalFile":
            file_queue.write(msg + "\n")
        elif self.queue_type == "AzureStorageQueue":
            self.queue_service.put_message(self.queue_name, msg)
        self.log.debug(msg)

    def dequeue(self):
        messages = []
        if self.queue_type == "LocalFile":
            with open(config.UNPROCESSED_LOG_FILE, "r") as f:
                messages = f.readlines()[1]
        elif self.queue_type == "AzureStorageQueue":
            messages = self.queue_service.get_messages(self.queue_name)
        return messages

    def delete(self, message):
        self.queue_service.delete_message(self.queue_name, message.message_id, message.pop_receipt)
        #  with open(config.PROCESSED_LOG_FILE, 'a') as processed:
        #    processed.write(log)
        #  os.remove(config.UNPROCESSED_LOG_FILE)

    def delete_queue(self, queue_name):
        self.queue_service.delete_queue(queue_name, False)

    def getLength(self):
        """
        Get the approximate length of the queue
        """
        queue_metadata = self.queue_service.get_queue_metadata(self.queue_name)
        count = queue_metadata["x-ms-approximate-messages-count"]
        return count
Beispiel #20
0
# Queue reader - Simple script to read and print entries in the queue
# Pedro Perez - 2015
#
# -----------------------------------------------------------------------------
import sys
import os
import subprocess
from azure.storage.queue import QueueService

account_name = "mcdockerqueue"
account_key = sys.argv[1]
queuename = "servers"

queue_service = QueueService(account_name, account_key)

queue_metadata = queue_service.get_queue_metadata(queuename)
count = queue_metadata['x-ms-approximate-messages-count']

print "There are %s messages in the queue" % count


if count > 0:
	messages = queue_service.get_messages(queuename)

	for message in messages:
	    print(message.message_text)

	# Remove message from the queue
	queue_service.delete_message(queuename, message.message_id, message.pop_receipt)
else:
	print "There are no messages to process"
        logger.warn(link)
        


if __name__ == '__main__':
    loggerName = "recognizeditems"
    logger = logging.getLogger(loggerName)

    azureQueueAccountName = ""
    azureQueueKey = ""
    azureQueueRecognizedItems = "recognizeditems-processing"
    queue_service = QueueService(account_name=azureQueueAccountName, account_key=azureQueueKey)
    try:

        while True:
            #get queue count
            metadata = queue_service.get_queue_metadata(azureQueueRecognizedItems)
            queue_count = metadata.approximate_message_count

            if queue_count > 0:
                read_next_in_queue()
            else:
                logger.info("time.sleep(3000)")
                time.sleep(3000)



    except Exception as e:
        init_logger()
        logger.error("Exception in main {}" .format(e))