def delete_storage_queue(self, resource_properties):
storage_client = StorageManagementClient(self.credentials, self.subscription_id)
key = storage_client.storage_accounts.list_keys(resource_properties['ResourceGroupName'], resource_properties['AccountName']).keys[0].value
queue_service = QueueService(account_name=resource_properties['AccountName'], account_key=key)
queue_service.delete_queue(resource_properties['Name'])
return {}
class AzureQueue(object):
def __init__(self, account_name, account_key, queue_name):
self.queue_name = queue_name
self.queue_service = QueueService(account_name=account_name, account_key=account_key)
self.queue_service.create_queue(self.queue_name)
def put_message_into_queue(self, content) -> QueueMessage:
"""
Publishes a message with `content`
:param content: The queue message
:returns: A QueueMessage that has the message as well as metadata
:rtype: QueueMessage
"""
return self.queue_service.put_message(self.queue_name, content)
def get_messages(self) -> list:
"""
Retrieves all of the messages that have been published into queue
:param content: The queue message
:returns: List of Queue messages
:rtype: list
"""
return self.queue_service.get_messages(self.queue_name)
def delete_message_from_queue(self, message_id, pop_receipt):
self.queue_service.delete_message(self.queue_name, message_id, pop_receipt)
def get_message_count(self):
queue_metadata = self.queue_service.get_queue_metadata(self.queue_name)
return queue_metadata.approximate_message_count
def delete(self):
return self.queue_service.delete_queue(self.queue_name)
def empty(self):
messages = queue_service.get_messages(self.queue_name,
num_messages=BATCH_NUMBER,
visibility_timeout=TIMEOUT_IN_SECONDS)
for message in messages:
self.queue_service.delete_message(self.queue_name, message.id, message.pop_receipt)
def delete_queue_from_storage_account(storage_account, name, session):
token = StorageUtilities.get_storage_token(session)
queue_service = QueueService(
account_name=storage_account.name,
token_credential=token)
return queue_service.delete_queue(name)
class Queue:
def __init__(self, account_name, account_key, queue_name="logqueue"):
"""Initialiaze a queue. The type is set by the
'ACS_LOGGING_QUEUE_TYPE' environment variable. If it is set to
'AzureStorageQueue' then values must be provided for
'account_name' and 'account_key' which are values associated
with the Azure Storage account. 'queue_name' is optional and
defaults to 'logqueue'.
"""
self.log = Log()
self.queue_type = config.ACS_LOGGING_QUEUE_TYPE
self.queue_name = queue_name
# self.log.debug("Queue type: " + self.queue_type + " / " + self.queue_name)
if self.queue_type == "AzureStorageQueue":
self.createAzureQueues(account_name, account_key)
elif self.queue_type == "LocalFile":
self.file_queue = open(config.UNPROCESSED_LOG_FILE, 'w+')
else:
self.log.error("Unknown queue type: " + queue_type)
def getName(self):
return self.queue_name
def createAzureQueues(self, account_name, account_key):
"""
Create a queue for unprocessed log messages. Entries in the queue
will be dequeued, processed and deleted upon success.
"""
self.queue_service = QueueService(account_name, account_key)
self.queue_service.create_queue(self.queue_name)
def close(self):
"""Perform any necessary clearnup on the queue
at the end of a run.
"""
if self.queue_type == "AzureStorageQueue":
pass
elif self.queue_type == "LocalFile":
self.file_queue.close()
else:
self.log.error("Unknown queue type: " + queue_type)
def enqueue(self, msg):
if self.queue_type == "LocalFile":
file_queue.write(msg + '\n')
elif self.queue_type == "AzureStorageQueue":
self.queue_service.put_message(self.queue_name, msg)
else:
self.log.error("We don't know how to handle queues of type " + self.queue_type)
self.log.debug(msg)
def dequeue(self):
messages = []
if self.queue_type == "LocalFile":
with open(config.UNPROCESSED_LOG_FILE, 'r') as f:
messages = f.readlines()[1]
elif self.queue_type == "AzureStorageQueue":
messages = self.queue_service.get_messages(self.queue_name)
return messages
def delete(self, message):
self.queue_service.delete_message(self.queue_name, message.message_id, message.pop_receipt)
# with open(config.PROCESSED_LOG_FILE, 'a') as processed:
# processed.write(log)
# os.remove(config.UNPROCESSED_LOG_FILE)
def delete_queue(self, queue_name):
self.queue_service.delete_queue(queue_name, False)
def getLength(self):
"""
Get the approximate length of the queue
"""
queue_metadata = self.queue_service.get_queue_metadata(self.queue_name)
count = queue_metadata['x-ms-approximate-messages-count']
return int(count)
def peek_messages(self, num_messages):
"""
Peek at the top messages in the queue. This method does not remove the
messages from the queue.
"""
return self.queue_service.peek_messages(self.queue_name, num_messages)
class AzureFunctionAppBackend:
"""
A wrap-up around Azure Function Apps backend.
"""
def __init__(self, config):
self.log_level = os.getenv('CLOUDBUTTON_LOGLEVEL')
self.name = 'azure_fa'
self.config = config
self.fa_client = FunctionAppClient(self.config)
self.queue_service = QueueService(
account_name=self.config['account_name'],
account_key=self.config['account_key'])
self.queue_service.encode_function = QueueMessageFormat.text_base64encode
self.queue_service.decode_function = QueueMessageFormat.text_base64decode
log_msg = 'Cloudbutton v{} init for Azure Function Apps'.format(
__version__)
logger.info(log_msg)
if not self.log_level:
print(log_msg)
def create_runtime(self,
docker_image_name,
memory=None,
timeout=azure_fa_config.RUNTIME_TIMEOUT_DEFAULT):
"""
Creates a new runtime into Azure Function Apps
from the provided Linux image for consumption plan
"""
log_msg = 'Creating new Cloudbutton runtime for Azure Function Apps...'
logger.info(log_msg)
if not self.log_level:
print(log_msg)
logger.info('Extracting preinstalls for Azure runtime')
metadata = self._generate_runtime_meta()
logger.info('Creating new Cloudbutton runtime')
action_name = self._format_action_name(docker_image_name)
self._create_runtime(action_name)
return metadata
def delete_runtime(self, docker_image_name, extract_preinstalls=False):
"""
Deletes a runtime
"""
if extract_preinstalls:
action_name = docker_image_name
else:
action_name = self._format_action_name(docker_image_name)
self.fa_client.delete_action(action_name)
queue_name = self._format_queue_name(docker_image_name, type='trigger')
self.queue_service.delete_queue(queue_name)
def invoke(self, docker_image_name, memory=None, payload={}):
"""
Invoke function
"""
action_name = self._format_action_name(docker_image_name)
queue_name = self._format_queue_name(action_name, type='trigger')
try:
msg = self.queue_service.put_message(queue_name,
json.dumps(payload))
activation_id = msg.id
except Exception:
logger.debug('Creating queue (invoke)')
self.queue_service.create_queue(queue_name)
return self.invoke(docker_image_name,
memory=memory,
payload=payload)
return activation_id
def get_runtime_key(self, docker_image_name, runtime_memory):
"""
Method that creates and returns the runtime key.
Runtime keys are used to uniquely identify runtimes within the storage,
in order to know which runtimes are installed and which not.
"""
action_name = self._format_action_name(docker_image_name)
runtime_key = os.path.join(self.name, action_name)
return runtime_key
def _format_action_name(self, action_name):
sha_1 = hashlib.sha1()
block = action_name.encode('ascii', errors='ignore')
sha_1.update(block)
tag = sha_1.hexdigest()[:8]
sha_1 = hashlib.sha1()
block = self.config['account_name'].encode('ascii', errors='ignore')
sha_1.update(block)
tag = tag + sha_1.hexdigest()[:8]
version = re.sub(r'[/_:.-]', '', __version__)
action_name = action_name[:16] + '-' + version[:5] + '-' + tag
return action_name
def _format_queue_name(self, action_name, type):
# Using different queue names because there is a delay between
# deleting a queue and creating another one with the same name
return action_name + '-' + type
def _create_runtime(self, action_name, extract_preinstalls=False):
"""
Creates a new runtime with the base modules and cloudbutton
"""
def add_base_modules():
cmd = 'pip3 install -t {} -r requirements.txt'.format(
azure_fa_config.ACTION_MODULES_DIR)
child = sp.Popen(cmd, shell=True, stdout=sp.PIPE,
stderr=sp.PIPE) # silent
child.wait()
logger.debug(child.stdout.read().decode())
logger.debug(child.stderr.read().decode())
if child.returncode != 0:
cmd = 'pip install -t {} -r requirements.txt'.format(
azure_fa_config.ACTION_MODULES_DIR)
child = sp.Popen(cmd,
shell=True,
stdout=sp.PIPE,
stderr=sp.PIPE) # silent
child.wait()
logger.debug(child.stdout.read().decode())
logger.debug(child.stderr.read().decode())
if child.returncode != 0:
logger.critical(
'Failed to install base modules for Azure Function')
exit(1)
def add_cloudbutton_module():
module_location = os.path.dirname(
os.path.abspath(cloudbutton.__file__))
shutil.copytree(
module_location,
os.path.join(azure_fa_config.ACTION_MODULES_DIR,
'cloudbutton'))
def get_bindings_str(action_name, extract_preinstalls=False):
if not extract_preinstalls:
bindings = {
"scriptFile":
"__init__.py",
"bindings": [{
"name":
"msgIn",
"type":
"queueTrigger",
"direction":
"in",
"queueName":
self._format_queue_name(action_name, 'trigger'),
"connection":
"AzureWebJobsStorage"
}]
}
else:
bindings = {
"scriptFile":
"__init__.py",
"bindings": [{
"name":
"msgIn",
"type":
"queueTrigger",
"direction":
"in",
"queueName":
self._format_queue_name(action_name, type='trigger'),
"connection":
"AzureWebJobsStorage"
}, {
"name":
"msgOut",
"type":
"queue",
"direction":
"out",
"queueName":
self._format_queue_name(action_name, type='result'),
"connection":
"AzureWebJobsStorage"
}]
}
return json.dumps(bindings)
initial_dir = os.getcwd()
temp_folder = next(tempfile._get_candidate_names())
os.mkdir(temp_folder)
os.chdir(temp_folder)
try:
# Create project folder from template
project_template = os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'action')
project_dir = os.path.join(initial_dir, temp_folder, action_name)
shutil.copytree(project_template, project_dir)
os.chdir(project_dir)
action_dir = os.path.join(project_dir, action_name)
os.rename('action', action_dir)
# Add the base dependencies and current cloudbutton module
logger.debug('Adding runtime base modules')
os.makedirs(azure_fa_config.ACTION_MODULES_DIR, exist_ok=True)
add_base_modules()
add_cloudbutton_module()
# Set entry point file
if extract_preinstalls:
entry_point_file = 'extract_preinstalls_action.py'
else:
entry_point_file = 'handler_action.py'
os.rename(os.path.join(action_dir, entry_point_file),
os.path.join(action_dir, '__init__.py'))
# Edit the function's bindings for it to be a queue triggered function
with open(os.path.join(action_dir, 'function.json'),
'w') as bindings_file:
bindings_file.write(
get_bindings_str(action_name, extract_preinstalls))
# Create trigger queue, create action
logger.debug('Creating trigger queue')
queue_name = self._format_queue_name(action_name, type='trigger')
self.queue_service.create_queue(queue_name)
self.fa_client.create_action(action_name)
except Exception as e:
raise Exception("Unable to create the new runtime", e)
finally:
os.chdir(initial_dir)
shutil.rmtree(temp_folder,
ignore_errors=True) # Remove tmp project folder
def _generate_runtime_meta(self):
"""
Extract installed Python modules from Azure runtime
"""
action_name = 'cloudbutton-extract-preinstalls-' + get_unique_id()
self._create_runtime(action_name, extract_preinstalls=True)
logger.debug("Invoking 'extract-preinstalls' action")
try:
runtime_meta = self._invoke_with_result(action_name)
except Exception:
raise Exception("Unable to invoke 'extract-preinstalls' action")
try:
self.delete_runtime(action_name, extract_preinstalls=True)
except Exception:
raise Exception("Unable to delete 'extract-preinstalls' action")
if not runtime_meta or 'preinstalls' not in runtime_meta:
raise Exception(runtime_meta)
logger.debug("Extracted metadata succesfully")
return runtime_meta
def _invoke_with_result(self, action_name):
result_queue_name = self._format_queue_name(action_name, type='result')
self.queue_service.create_queue(result_queue_name)
trigger_queue_name = self._format_queue_name(action_name,
type='trigger')
self.queue_service.put_message(trigger_queue_name, '')
msg = []
while not msg:
msg = self.queue_service.get_messages(result_queue_name,
num_messages=1)
time.sleep(0.5)
result_str = msg[0].content
self.queue_service.delete_queue(result_queue_name)
return json.loads(result_str)
metadata = queue_service.get_queue_metadata('pizzaqueue')
print(
'If we look at the Queue again, we have one less message to show we have processed that order and a yummy pizza will be on it\'s way to the customer soon.'
)
print(('Number of messages in the queue: ' +
str(metadata.approximate_message_count)))
input('\nPress Enter to continue...')
###
# This was a quick demo to see Queues in action.
# Although the actual cost is minimal since we deleted all the messages from the Queue, it's good to clean up resources when you're done
###
print(
'\nThis is a basic example of how Azure Storage Queues behave.\nTo keep things tidy, let\'s clean up the Azure Storage resources we created.'
)
input('Press Enter to continue...')
response = queue_service.delete_queue('pizzaqueue')
if response == True:
print('Storage Queue: pizzaqueue deleted successfully.')
else:
print('Error deleting Storage Queue')
response = azurerm.delete_resource_group(auth_token, subscription_id,
resourcegroup_name)
if response.status_code == 202:
print(('Resource group: ' + resourcegroup_name + ' deleted successfully.'))
else:
print('Error deleting resource group.')
class Queue:
def __init__(self, account_name, account_key, queue_name="logqueue"):
"""Initialiaze a queue. The type is set by the
'ACS_LOGGING_QUEUE_TYPE' environment variable. If it is set to
'AzureStorageQueue' then values must be provided for
'account_name' and 'account_key' which are values associated
with the Azure Storage account. 'queue_name' is optional and
defaults to 'logqueue'.
"""
self.log = Log()
self.queue_type = config.ACS_LOGGING_QUEUE_TYPE
self.queue_name = queue_name
# self.log.debug("Queue type: " + self.queue_type + " / " + self.queue_name)
if self.queue_type == "AzureStorageQueue":
self.createAzureQueues(account_name, account_key)
elif self.queue_type == "LocalFile":
self.file_queue = open(config.UNPROCESSED_LOG_FILE, "w+")
else:
self.log.error("Unknown queue type: " + queue_type)
def createAzureQueues(self, account_name, account_key):
"""
Create a queue for unprocessed log messages. Entries in the queue
will be dequeued, processed and deleted upon success.
"""
self.queue_service = QueueService(account_name, account_key)
self.queue_service.create_queue(self.queue_name)
def close(self):
"""Perform any necessary clearnup on the queue
at the end of a run.
"""
if self.queue_type == "AzureStorageQueue":
pass
elif self.queue_type == "LocalFile":
self.file_queue.close()
else:
self.log.error("Unknown queue type: " + queue_type)
def enqueue(self, msg, level="INFO"):
msg = level + " - " + msg
if self.queue_type == "LocalFile":
file_queue.write(msg + "\n")
elif self.queue_type == "AzureStorageQueue":
self.queue_service.put_message(self.queue_name, msg)
self.log.debug(msg)
def dequeue(self):
messages = []
if self.queue_type == "LocalFile":
with open(config.UNPROCESSED_LOG_FILE, "r") as f:
messages = f.readlines()[1]
elif self.queue_type == "AzureStorageQueue":
messages = self.queue_service.get_messages(self.queue_name)
return messages
def delete(self, message):
self.queue_service.delete_message(self.queue_name, message.message_id, message.pop_receipt)
# with open(config.PROCESSED_LOG_FILE, 'a') as processed:
# processed.write(log)
# os.remove(config.UNPROCESSED_LOG_FILE)
def delete_queue(self, queue_name):
self.queue_service.delete_queue(queue_name, False)
def getLength(self):
"""
Get the approximate length of the queue
"""
queue_metadata = self.queue_service.get_queue_metadata(self.queue_name)
count = queue_metadata["x-ms-approximate-messages-count"]
return count
)
sleep(6)
else:
print("\n\nSuccessfully deleted message!\n")
finally:
messages = queue_service.get_messages('umairqueue', num_messages=1)
for message in messages:
print(message.content)
queue_service.delete_message('umairqueue', message.id,
message.pop_receipt)
input(
"\n\nLeave one message in the queue and do not delete the queue right away.\n Press any key to contine..."
)
metadata = queue_service.get_queue_metadata('umairqueue')
while (metadata.approximate_message_count > 1):
messages = queue_service.get_messages('umairqueue', num_messages=1)
for message in messages:
print("Deleting message: " + message.content + '\n\n')
queue_service.delete_message('umairqueue', message.id,
message.pop_receipt)
metadata = queue_service.get_queue_metadata('umairqueue')
message = queue_service.peek_messages('umairqueue')
print("Last message left in the queue: " + message[0].content + "\n\n")
input("Deleting queue, press any key to proceed?\n")
queue_service.delete_queue('umairqueue')
print("Queue deleted successfully!")
