def delete_storage_queue(self, resource_properties): storage_client = StorageManagementClient(self.credentials, self.subscription_id) key = storage_client.storage_accounts.list_keys(resource_properties['ResourceGroupName'], resource_properties['AccountName']).keys[0].value queue_service = QueueService(account_name=resource_properties['AccountName'], account_key=key) queue_service.delete_queue(resource_properties['Name']) return {}
class AzureQueue(object): def __init__(self, account_name, account_key, queue_name): self.queue_name = queue_name self.queue_service = QueueService(account_name=account_name, account_key=account_key) self.queue_service.create_queue(self.queue_name) def put_message_into_queue(self, content) -> QueueMessage: """ Publishes a message with `content` :param content: The queue message :returns: A QueueMessage that has the message as well as metadata :rtype: QueueMessage """ return self.queue_service.put_message(self.queue_name, content) def get_messages(self) -> list: """ Retrieves all of the messages that have been published into queue :param content: The queue message :returns: List of Queue messages :rtype: list """ return self.queue_service.get_messages(self.queue_name) def delete_message_from_queue(self, message_id, pop_receipt): self.queue_service.delete_message(self.queue_name, message_id, pop_receipt) def get_message_count(self): queue_metadata = self.queue_service.get_queue_metadata(self.queue_name) return queue_metadata.approximate_message_count def delete(self): return self.queue_service.delete_queue(self.queue_name) def empty(self): messages = queue_service.get_messages(self.queue_name, num_messages=BATCH_NUMBER, visibility_timeout=TIMEOUT_IN_SECONDS) for message in messages: self.queue_service.delete_message(self.queue_name, message.id, message.pop_receipt)
def delete_queue_from_storage_account(storage_account, name, session): token = StorageUtilities.get_storage_token(session) queue_service = QueueService( account_name=storage_account.name, token_credential=token) return queue_service.delete_queue(name)
class Queue: def __init__(self, account_name, account_key, queue_name="logqueue"): """Initialiaze a queue. The type is set by the 'ACS_LOGGING_QUEUE_TYPE' environment variable. If it is set to 'AzureStorageQueue' then values must be provided for 'account_name' and 'account_key' which are values associated with the Azure Storage account. 'queue_name' is optional and defaults to 'logqueue'. """ self.log = Log() self.queue_type = config.ACS_LOGGING_QUEUE_TYPE self.queue_name = queue_name # self.log.debug("Queue type: " + self.queue_type + " / " + self.queue_name) if self.queue_type == "AzureStorageQueue": self.createAzureQueues(account_name, account_key) elif self.queue_type == "LocalFile": self.file_queue = open(config.UNPROCESSED_LOG_FILE, 'w+') else: self.log.error("Unknown queue type: " + queue_type) def getName(self): return self.queue_name def createAzureQueues(self, account_name, account_key): """ Create a queue for unprocessed log messages. Entries in the queue will be dequeued, processed and deleted upon success. """ self.queue_service = QueueService(account_name, account_key) self.queue_service.create_queue(self.queue_name) def close(self): """Perform any necessary clearnup on the queue at the end of a run. """ if self.queue_type == "AzureStorageQueue": pass elif self.queue_type == "LocalFile": self.file_queue.close() else: self.log.error("Unknown queue type: " + queue_type) def enqueue(self, msg): if self.queue_type == "LocalFile": file_queue.write(msg + '\n') elif self.queue_type == "AzureStorageQueue": self.queue_service.put_message(self.queue_name, msg) else: self.log.error("We don't know how to handle queues of type " + self.queue_type) self.log.debug(msg) def dequeue(self): messages = [] if self.queue_type == "LocalFile": with open(config.UNPROCESSED_LOG_FILE, 'r') as f: messages = f.readlines()[1] elif self.queue_type == "AzureStorageQueue": messages = self.queue_service.get_messages(self.queue_name) return messages def delete(self, message): self.queue_service.delete_message(self.queue_name, message.message_id, message.pop_receipt) # with open(config.PROCESSED_LOG_FILE, 'a') as processed: # processed.write(log) # os.remove(config.UNPROCESSED_LOG_FILE) def delete_queue(self, queue_name): self.queue_service.delete_queue(queue_name, False) def getLength(self): """ Get the approximate length of the queue """ queue_metadata = self.queue_service.get_queue_metadata(self.queue_name) count = queue_metadata['x-ms-approximate-messages-count'] return int(count) def peek_messages(self, num_messages): """ Peek at the top messages in the queue. This method does not remove the messages from the queue. """ return self.queue_service.peek_messages(self.queue_name, num_messages)
class AzureFunctionAppBackend: """ A wrap-up around Azure Function Apps backend. """ def __init__(self, config): self.log_level = os.getenv('CLOUDBUTTON_LOGLEVEL') self.name = 'azure_fa' self.config = config self.fa_client = FunctionAppClient(self.config) self.queue_service = QueueService( account_name=self.config['account_name'], account_key=self.config['account_key']) self.queue_service.encode_function = QueueMessageFormat.text_base64encode self.queue_service.decode_function = QueueMessageFormat.text_base64decode log_msg = 'Cloudbutton v{} init for Azure Function Apps'.format( __version__) logger.info(log_msg) if not self.log_level: print(log_msg) def create_runtime(self, docker_image_name, memory=None, timeout=azure_fa_config.RUNTIME_TIMEOUT_DEFAULT): """ Creates a new runtime into Azure Function Apps from the provided Linux image for consumption plan """ log_msg = 'Creating new Cloudbutton runtime for Azure Function Apps...' logger.info(log_msg) if not self.log_level: print(log_msg) logger.info('Extracting preinstalls for Azure runtime') metadata = self._generate_runtime_meta() logger.info('Creating new Cloudbutton runtime') action_name = self._format_action_name(docker_image_name) self._create_runtime(action_name) return metadata def delete_runtime(self, docker_image_name, extract_preinstalls=False): """ Deletes a runtime """ if extract_preinstalls: action_name = docker_image_name else: action_name = self._format_action_name(docker_image_name) self.fa_client.delete_action(action_name) queue_name = self._format_queue_name(docker_image_name, type='trigger') self.queue_service.delete_queue(queue_name) def invoke(self, docker_image_name, memory=None, payload={}): """ Invoke function """ action_name = self._format_action_name(docker_image_name) queue_name = self._format_queue_name(action_name, type='trigger') try: msg = self.queue_service.put_message(queue_name, json.dumps(payload)) activation_id = msg.id except Exception: logger.debug('Creating queue (invoke)') self.queue_service.create_queue(queue_name) return self.invoke(docker_image_name, memory=memory, payload=payload) return activation_id def get_runtime_key(self, docker_image_name, runtime_memory): """ Method that creates and returns the runtime key. Runtime keys are used to uniquely identify runtimes within the storage, in order to know which runtimes are installed and which not. """ action_name = self._format_action_name(docker_image_name) runtime_key = os.path.join(self.name, action_name) return runtime_key def _format_action_name(self, action_name): sha_1 = hashlib.sha1() block = action_name.encode('ascii', errors='ignore') sha_1.update(block) tag = sha_1.hexdigest()[:8] sha_1 = hashlib.sha1() block = self.config['account_name'].encode('ascii', errors='ignore') sha_1.update(block) tag = tag + sha_1.hexdigest()[:8] version = re.sub(r'[/_:.-]', '', __version__) action_name = action_name[:16] + '-' + version[:5] + '-' + tag return action_name def _format_queue_name(self, action_name, type): # Using different queue names because there is a delay between # deleting a queue and creating another one with the same name return action_name + '-' + type def _create_runtime(self, action_name, extract_preinstalls=False): """ Creates a new runtime with the base modules and cloudbutton """ def add_base_modules(): cmd = 'pip3 install -t {} -r requirements.txt'.format( azure_fa_config.ACTION_MODULES_DIR) child = sp.Popen(cmd, shell=True, stdout=sp.PIPE, stderr=sp.PIPE) # silent child.wait() logger.debug(child.stdout.read().decode()) logger.debug(child.stderr.read().decode()) if child.returncode != 0: cmd = 'pip install -t {} -r requirements.txt'.format( azure_fa_config.ACTION_MODULES_DIR) child = sp.Popen(cmd, shell=True, stdout=sp.PIPE, stderr=sp.PIPE) # silent child.wait() logger.debug(child.stdout.read().decode()) logger.debug(child.stderr.read().decode()) if child.returncode != 0: logger.critical( 'Failed to install base modules for Azure Function') exit(1) def add_cloudbutton_module(): module_location = os.path.dirname( os.path.abspath(cloudbutton.__file__)) shutil.copytree( module_location, os.path.join(azure_fa_config.ACTION_MODULES_DIR, 'cloudbutton')) def get_bindings_str(action_name, extract_preinstalls=False): if not extract_preinstalls: bindings = { "scriptFile": "__init__.py", "bindings": [{ "name": "msgIn", "type": "queueTrigger", "direction": "in", "queueName": self._format_queue_name(action_name, 'trigger'), "connection": "AzureWebJobsStorage" }] } else: bindings = { "scriptFile": "__init__.py", "bindings": [{ "name": "msgIn", "type": "queueTrigger", "direction": "in", "queueName": self._format_queue_name(action_name, type='trigger'), "connection": "AzureWebJobsStorage" }, { "name": "msgOut", "type": "queue", "direction": "out", "queueName": self._format_queue_name(action_name, type='result'), "connection": "AzureWebJobsStorage" }] } return json.dumps(bindings) initial_dir = os.getcwd() temp_folder = next(tempfile._get_candidate_names()) os.mkdir(temp_folder) os.chdir(temp_folder) try: # Create project folder from template project_template = os.path.join( os.path.dirname(os.path.abspath(__file__)), 'action') project_dir = os.path.join(initial_dir, temp_folder, action_name) shutil.copytree(project_template, project_dir) os.chdir(project_dir) action_dir = os.path.join(project_dir, action_name) os.rename('action', action_dir) # Add the base dependencies and current cloudbutton module logger.debug('Adding runtime base modules') os.makedirs(azure_fa_config.ACTION_MODULES_DIR, exist_ok=True) add_base_modules() add_cloudbutton_module() # Set entry point file if extract_preinstalls: entry_point_file = 'extract_preinstalls_action.py' else: entry_point_file = 'handler_action.py' os.rename(os.path.join(action_dir, entry_point_file), os.path.join(action_dir, '__init__.py')) # Edit the function's bindings for it to be a queue triggered function with open(os.path.join(action_dir, 'function.json'), 'w') as bindings_file: bindings_file.write( get_bindings_str(action_name, extract_preinstalls)) # Create trigger queue, create action logger.debug('Creating trigger queue') queue_name = self._format_queue_name(action_name, type='trigger') self.queue_service.create_queue(queue_name) self.fa_client.create_action(action_name) except Exception as e: raise Exception("Unable to create the new runtime", e) finally: os.chdir(initial_dir) shutil.rmtree(temp_folder, ignore_errors=True) # Remove tmp project folder def _generate_runtime_meta(self): """ Extract installed Python modules from Azure runtime """ action_name = 'cloudbutton-extract-preinstalls-' + get_unique_id() self._create_runtime(action_name, extract_preinstalls=True) logger.debug("Invoking 'extract-preinstalls' action") try: runtime_meta = self._invoke_with_result(action_name) except Exception: raise Exception("Unable to invoke 'extract-preinstalls' action") try: self.delete_runtime(action_name, extract_preinstalls=True) except Exception: raise Exception("Unable to delete 'extract-preinstalls' action") if not runtime_meta or 'preinstalls' not in runtime_meta: raise Exception(runtime_meta) logger.debug("Extracted metadata succesfully") return runtime_meta def _invoke_with_result(self, action_name): result_queue_name = self._format_queue_name(action_name, type='result') self.queue_service.create_queue(result_queue_name) trigger_queue_name = self._format_queue_name(action_name, type='trigger') self.queue_service.put_message(trigger_queue_name, '') msg = [] while not msg: msg = self.queue_service.get_messages(result_queue_name, num_messages=1) time.sleep(0.5) result_str = msg[0].content self.queue_service.delete_queue(result_queue_name) return json.loads(result_str)
metadata = queue_service.get_queue_metadata('pizzaqueue') print( 'If we look at the Queue again, we have one less message to show we have processed that order and a yummy pizza will be on it\'s way to the customer soon.' ) print(('Number of messages in the queue: ' + str(metadata.approximate_message_count))) input('\nPress Enter to continue...') ### # This was a quick demo to see Queues in action. # Although the actual cost is minimal since we deleted all the messages from the Queue, it's good to clean up resources when you're done ### print( '\nThis is a basic example of how Azure Storage Queues behave.\nTo keep things tidy, let\'s clean up the Azure Storage resources we created.' ) input('Press Enter to continue...') response = queue_service.delete_queue('pizzaqueue') if response == True: print('Storage Queue: pizzaqueue deleted successfully.') else: print('Error deleting Storage Queue') response = azurerm.delete_resource_group(auth_token, subscription_id, resourcegroup_name) if response.status_code == 202: print(('Resource group: ' + resourcegroup_name + ' deleted successfully.')) else: print('Error deleting resource group.')
class Queue: def __init__(self, account_name, account_key, queue_name="logqueue"): """Initialiaze a queue. The type is set by the 'ACS_LOGGING_QUEUE_TYPE' environment variable. If it is set to 'AzureStorageQueue' then values must be provided for 'account_name' and 'account_key' which are values associated with the Azure Storage account. 'queue_name' is optional and defaults to 'logqueue'. """ self.log = Log() self.queue_type = config.ACS_LOGGING_QUEUE_TYPE self.queue_name = queue_name # self.log.debug("Queue type: " + self.queue_type + " / " + self.queue_name) if self.queue_type == "AzureStorageQueue": self.createAzureQueues(account_name, account_key) elif self.queue_type == "LocalFile": self.file_queue = open(config.UNPROCESSED_LOG_FILE, "w+") else: self.log.error("Unknown queue type: " + queue_type) def createAzureQueues(self, account_name, account_key): """ Create a queue for unprocessed log messages. Entries in the queue will be dequeued, processed and deleted upon success. """ self.queue_service = QueueService(account_name, account_key) self.queue_service.create_queue(self.queue_name) def close(self): """Perform any necessary clearnup on the queue at the end of a run. """ if self.queue_type == "AzureStorageQueue": pass elif self.queue_type == "LocalFile": self.file_queue.close() else: self.log.error("Unknown queue type: " + queue_type) def enqueue(self, msg, level="INFO"): msg = level + " - " + msg if self.queue_type == "LocalFile": file_queue.write(msg + "\n") elif self.queue_type == "AzureStorageQueue": self.queue_service.put_message(self.queue_name, msg) self.log.debug(msg) def dequeue(self): messages = [] if self.queue_type == "LocalFile": with open(config.UNPROCESSED_LOG_FILE, "r") as f: messages = f.readlines()[1] elif self.queue_type == "AzureStorageQueue": messages = self.queue_service.get_messages(self.queue_name) return messages def delete(self, message): self.queue_service.delete_message(self.queue_name, message.message_id, message.pop_receipt) # with open(config.PROCESSED_LOG_FILE, 'a') as processed: # processed.write(log) # os.remove(config.UNPROCESSED_LOG_FILE) def delete_queue(self, queue_name): self.queue_service.delete_queue(queue_name, False) def getLength(self): """ Get the approximate length of the queue """ queue_metadata = self.queue_service.get_queue_metadata(self.queue_name) count = queue_metadata["x-ms-approximate-messages-count"] return count
) sleep(6) else: print("\n\nSuccessfully deleted message!\n") finally: messages = queue_service.get_messages('umairqueue', num_messages=1) for message in messages: print(message.content) queue_service.delete_message('umairqueue', message.id, message.pop_receipt) input( "\n\nLeave one message in the queue and do not delete the queue right away.\n Press any key to contine..." ) metadata = queue_service.get_queue_metadata('umairqueue') while (metadata.approximate_message_count > 1): messages = queue_service.get_messages('umairqueue', num_messages=1) for message in messages: print("Deleting message: " + message.content + '\n\n') queue_service.delete_message('umairqueue', message.id, message.pop_receipt) metadata = queue_service.get_queue_metadata('umairqueue') message = queue_service.peek_messages('umairqueue') print("Last message left in the queue: " + message[0].content + "\n\n") input("Deleting queue, press any key to proceed?\n") queue_service.delete_queue('umairqueue') print("Queue deleted successfully!")