def create_queues(self) -> None: logger.info("creating eventgrid destination queue") name = self.results["deploy"]["func-name"]["value"] key = self.results["deploy"]["func-key"]["value"] account_url = "https://%s.queue.core.windows.net" % name client = QueueServiceClient( account_url=account_url, credential={ "account_name": name, "account_key": key }, ) for queue in [ "file-changes", "task-heartbeat", "node-heartbeat", "proxy", "update-queue", "webhooks", ]: try: client.create_queue(queue) except ResourceExistsError: pass
def create_queue(queue_name, clear_queue): credential = DefaultAzureCredential() account_url = "https://thecupstore.queue.core.windows.net/" queueservice = QueueServiceClient(account_url=account_url, credential=credential) try: queueservice.create_queue(name=queue_name) except: # Check that exists and clear if chosen if clear_queue: for queue in queueservice.list_queues(): if queue["name"] == queue_name: queue_client = get_queue(queue_name) queue_client.clear_messages() break
def start_round(): matches = flask.request.get_json( ) # requires header content-type of "application/json" keyVaultName = os.environ["KEY_VAULT_NAME"] keyVault_URI = "https://" + keyVaultName + ".vault.azure.net" credential = DefaultAzureCredential() client = SecretClient(vault_url=keyVault_URI, credential=credential) data_access_key = client.get_secret("thecupstore-key") table_service = TableService(account_name='thecupstore', account_key=data_access_key.value) # Create the query string. Expects a list of matches, each of which is list containing 2 teams. query_string = "" for match in matches: for team in match: query_string += "Name eq \'" + team + "\' or " # Remove trailing ' or ' query_string = query_string[:-4] team_stats = table_service.query_entities('Teams', filter=query_string) global current_round current_round = classes.round.Round(matches, team_stats) # Create the message queue for sending goal updates queue_name = "goalqueue" account_url = "https://thecupstore.queue.core.windows.net/" queueservice = QueueServiceClient(account_url=account_url, credential=credential) queueservice.create_queue(name=queue_name) global queueclient queueclient = QueueClient(account_url=account_url, queue_name=queue_name, credential=data_access_key.value) return '', 200
def test_request_callback_signed_header(self, resource_group, location, storage_account, storage_account_key): # Arrange service = QueueServiceClient(self.account_url(storage_account, "queue"), credential=storage_account_key) name = self.get_resource_name('cont') # Act try: headers = {'x-ms-meta-hello': 'world'} queue = service.create_queue(name, headers=headers) # Assert metadata = queue.get_queue_properties().metadata self.assertEqual(metadata, {'hello': 'world'}) finally: service.delete_queue(name)
class AzureFunctionAppBackend: """ A wrap-up around Azure Function Apps backend. """ def __init__(self, config, storage_config): logger.debug("Creating Azure Functions client") self.name = 'azure_fa' self.azure_config = config self.resource_group = self.azure_config['resource_group'] self.storage_account = self.azure_config['storage_account'] self.account_key = self.azure_config['storage_account_key'] self.location = self.azure_config['location'] self.functions_version = self.azure_config['functions_version'] self.queue_service_url = 'https://{}.queue.core.windows.net'.format(self.storage_account) self.queue_service = QueueServiceClient(account_url=self.queue_service_url, credential=self.account_key) msg = COMPUTE_CLI_MSG.format('Azure Functions') logger.info("{} - Location: {}".format(msg, self.location)) def _format_action_name(self, runtime_name, runtime_memory=None): runtime_name = runtime_name.replace('/', '--').replace(':', '--') return runtime_name def _format_queue_name(self, action_name, q_type): runtime_name = action_name.replace('--', '-') return runtime_name+'-'+q_type def _get_default_runtime_image_name(self): py_version = version_str(sys.version_info).replace('.', '') revision = 'latest' if 'dev' in __version__ else __version__.replace('.', '') runtime_name = '{}-{}-v{}-{}'.format(self.storage_account, az_config.RUNTIME_NAME, py_version, revision) return runtime_name def create_runtime(self, docker_image_name, memory=None, timeout=az_config.RUNTIME_TIMEOUT): """ Creates a new runtime into Azure Function Apps from the provided Linux image for consumption plan """ default_runtime_img_name = self._get_default_runtime_image_name() if docker_image_name in ['default', default_runtime_img_name]: # We only build the default image. rest of images must already exist # in the docker registry. docker_image_name = default_runtime_img_name self._build_default_runtime(default_runtime_img_name) logger.info('Creating new Lithops runtime for Azure Function Apps') self._create_function(docker_image_name, memory, timeout) metadata = self._generate_runtime_meta(docker_image_name, memory) return metadata def _build_default_runtime(self, default_runtime_img_name): """ Builds the default runtime """ return self.build_runtime(default_runtime_img_name) if os.system('{} --version >{} 2>&1'.format(az_config.DOCKER_PATH, os.devnull)) == 0: # Build default runtime using local dokcer python_version = version_str(sys.version_info) dockerfile = "Dockefile.default-azure-runtime" with open(dockerfile, 'w') as f: f.write("FROM mcr.microsoft.com/azure-functions/python:3.0-python{}\n".format(python_version)) f.write(az_config.DEFAULT_DOCKERFILE) self.build_runtime_docker(default_runtime_img_name, dockerfile) os.remove(dockerfile) else: raise Exception('docker command not found. Install docker or use ' 'an already built runtime') def build_runtime(self, runtime_name, requirements_file=None): try: shutil.rmtree(az_config.BUILD_DIR) except Exception: pass action_name = self._format_action_name(runtime_name) build_dir = os.path.join(az_config.BUILD_DIR, action_name) os.makedirs(build_dir, exist_ok=True) logger.info('Building default runtime in {}'.format(build_dir)) action_dir = os.path.join(build_dir, az_config.ACTION_DIR) os.makedirs(action_dir, exist_ok=True) req_file = os.path.join(build_dir, 'requirements.txt') with open(req_file, 'w') as reqf: reqf.write(az_config.REQUIREMENTS_FILE) host_file = os.path.join(build_dir, 'host.json') with open(host_file, 'w') as hstf: hstf.write(az_config.HOST_FILE) fn_file = os.path.join(action_dir, 'function.json') with open(fn_file, 'w') as fnf: in_q_name = self._format_queue_name(action_name, az_config.IN_QUEUE) az_config.BINDINGS['bindings'][0]['queueName'] = in_q_name out_q_name = self._format_queue_name(action_name, az_config.OUT_QUEUE) az_config.BINDINGS['bindings'][1]['queueName'] = out_q_name fnf.write(json.dumps(az_config.BINDINGS)) entry_point = os.path.join(os.path.dirname(__file__), 'entry_point.py') main_file = os.path.join(action_dir, '__init__.py') shutil.copy(entry_point, main_file) mod_dir = os.path.join(build_dir, az_config.ACTION_MODULES_DIR) os.chdir(build_dir) cmd = 'pip3 install -U -t {} -r requirements.txt'.format(mod_dir) if logger.getEffectiveLevel() != logging.DEBUG: cmd = cmd + " >{} 2>&1".format(os.devnull) os.system(cmd) lithops_location = os.path.dirname(os.path.abspath(lithops.__file__)) shutil.copytree(lithops_location, os.path.join(mod_dir, 'lithops')) def build_runtime_docker(self, docker_image_name, dockerfile): """ Builds a new runtime from a Docker file and pushes it to the Docker hub """ logger.debug('Building new docker image from Dockerfile') logger.debug('Docker image name: {}'.format(docker_image_name)) entry_point = os.path.join(os.path.dirname(__file__), 'entry_point.py') create_handler_zip(az_config.FH_ZIP_LOCATION, entry_point, '__init__.py') if dockerfile: cmd = '{} build -t {} -f {} .'.format(az_config.DOCKER_PATH, docker_image_name, dockerfile) else: cmd = '{} build -t {} .'.format(az_config.DOCKER_PATH, docker_image_name) if logger.getEffectiveLevel() != logging.DEBUG: cmd = cmd + " >{} 2>&1".format(os.devnull) logger.info('Building default runtime') res = os.system(cmd) if res != 0: raise Exception('There was an error building the runtime') cmd = '{} push {}'.format(az_config.DOCKER_PATH, docker_image_name) if logger.getEffectiveLevel() != logging.DEBUG: cmd = cmd + " >{} 2>&1".format(os.devnull) res = os.system(cmd) if res != 0: raise Exception('There was an error pushing the runtime to the container registry') logger.debug('Done!') def _create_function(self, docker_image_name, memory=None, timeout=az_config.RUNTIME_TIMEOUT): """ Create and publish an Azure Function App """ action_name = self._format_action_name(docker_image_name, memory) try: in_q_name = self._format_queue_name(action_name, az_config.IN_QUEUE) self.queue_service.create_queue(in_q_name) except Exception: in_queue = self.queue_service.get_queue_client(in_q_name) in_queue.clear_messages() try: out_q_name = self._format_queue_name(action_name, az_config.OUT_QUEUE) self.queue_service.create_queue(out_q_name) except Exception: out_queue = self.queue_service.get_queue_client(out_q_name) out_queue.clear_messages() logger.debug('Creating function app') logger.debug('Function name: {}'.format(action_name)) python_version = version_str(sys.version_info) cmd = ('az functionapp create --name {} --storage-account {} ' '--resource-group {} --os-type Linux --runtime python ' '--runtime-version {} --functions-version {} --consumption-plan-location {}' .format(action_name, self.storage_account, self.resource_group, python_version, self.functions_version, self.location)) if logger.getEffectiveLevel() != logging.DEBUG: cmd = cmd + " >{} 2>&1".format(os.devnull) res = os.system(cmd) if res != 0: raise Exception('There was an error creating the function in Azure. cmd: {}'.format(cmd)) logger.debug('Publishing function app') build_dir = os.path.join(az_config.BUILD_DIR, action_name) os.chdir(build_dir) res = 1 while res != 0: time.sleep(5) cmd = 'func azure functionapp publish {} --python --no-build'.format(action_name) if logger.getEffectiveLevel() != logging.DEBUG: cmd = cmd + " >{} 2>&1".format(os.devnull) res = os.system(cmd) time.sleep(10) def delete_runtime(self, runtime_name, memory): """ Deletes a runtime """ action_name = self._format_action_name(runtime_name, memory) logger.debug('Deleting function app: {}'.format(action_name)) cmd = ('az functionapp delete --name {} --resource-group {}' .format(action_name, self.resource_group)) if logger.getEffectiveLevel() != logging.DEBUG: cmd = cmd + " >{} 2>&1".format(os.devnull) try: in_q_name = self._format_queue_name(action_name, az_config.IN_QUEUE) self.queue_service.delete_queue(in_q_name) except Exception: pass try: out_q_name = self._format_queue_name(action_name, az_config.OUT_QUEUE) self.queue_service.delete_queue(out_q_name) except Exception: pass def invoke(self, docker_image_name, memory=None, payload={}, return_result=False): """ Invoke function """ action_name = self._format_action_name(docker_image_name) in_q_name = self._format_queue_name(action_name, az_config.IN_QUEUE) in_queue = self.queue_service.get_queue_client(in_q_name) msg = in_queue.send_message(dict_to_b64str(payload)) activation_id = msg.id if return_result: out_q_name = self._format_queue_name(action_name, az_config.OUT_QUEUE) out_queue = self.queue_service.get_queue_client(out_q_name) msg = [] while not msg: time.sleep(1) msg = out_queue.receive_message() out_queue.clear_messages() return b64str_to_dict(msg.content) return activation_id def get_runtime_key(self, docker_image_name, runtime_memory): """ Method that creates and returns the runtime key. Runtime keys are used to uniquely identify runtimes within the storage, in order to know which runtimes are installed and which not. """ action_name = self._format_action_name(docker_image_name, runtime_memory) runtime_key = os.path.join(self.name, action_name) return runtime_key def clean(self): # TODO pass def _generate_runtime_meta(self, docker_image_name, memory): """ Extract installed Python modules from Azure runtime """ logger.info("Extracting Python modules from: {}".format(docker_image_name)) payload = {'log_level': logger.getEffectiveLevel(), 'get_preinstalls': True} try: runtime_meta = self.invoke(docker_image_name, memory=memory, payload=payload, return_result=True) except Exception: raise Exception("Unable to invoke 'extract-preinstalls' action") if not runtime_meta or 'preinstalls' not in runtime_meta: raise Exception(runtime_meta) logger.debug("Extracted metadata succesfully") return runtime_meta
class AzureFunctionAppBackend: """ A wrap-up around Azure Function Apps backend. """ def __init__(self, config, internal_storage): logger.debug("Creating Azure Functions client") self.name = 'azure_fa' self.type = 'faas' self.azure_config = config self.invocation_type = self.azure_config['invocation_type'] self.resource_group = self.azure_config['resource_group'] self.storage_account_name = self.azure_config['storage_account_name'] self.storage_account_key = self.azure_config['storage_account_key'] self.location = self.azure_config['location'] self.functions_version = self.azure_config['functions_version'] self.queue_service_url = 'https://{}.queue.core.windows.net'.format( self.storage_account_name) self.queue_service = QueueServiceClient( account_url=self.queue_service_url, credential=self.storage_account_key) msg = COMPUTE_CLI_MSG.format('Azure Functions') logger.info("{} - Location: {}".format(msg, self.location)) def _format_action_name(self, runtime_name, runtime_memory=None): runtime_name = runtime_name.replace('/', '--').replace(':', '--') return runtime_name def _format_queue_name(self, action_name, q_type): runtime_name = action_name.replace('--', '-') return runtime_name + '-' + q_type def _get_default_runtime_image_name(self): py_version = version_str(sys.version_info).replace('.', '') revision = 'latest' if 'dev' in __version__ else __version__.replace( '.', '') runtime_name = '{}-{}-v{}-{}-{}'.format(self.storage_account_name, az_config.RUNTIME_NAME, py_version, revision, self.invocation_type) return runtime_name def create_runtime(self, docker_image_name, memory, timeout): """ Creates a new runtime into Azure Function Apps from the provided Linux image for consumption plan """ default_runtime_img_name = self._get_default_runtime_image_name() if docker_image_name in ['default', default_runtime_img_name]: # We only build the default image. rest of images must already exist # in the docker registry. docker_image_name = default_runtime_img_name self._build_default_runtime(default_runtime_img_name) self._create_function(docker_image_name, memory, timeout) metadata = self._generate_runtime_meta(docker_image_name, memory) return metadata def _build_default_runtime(self, default_runtime_img_name): """ Builds the default runtime """ return self.build_runtime(default_runtime_img_name) if os.system('{} --version >{} 2>&1'.format(az_config.DOCKER_PATH, os.devnull)) == 0: # Build default runtime using local dokcer python_version = version_str(sys.version_info) dockerfile = "Dockefile.default-azure-runtime" with open(dockerfile, 'w') as f: f.write( "FROM mcr.microsoft.com/azure-functions/python:3.0-python{}\n" .format(python_version)) f.write(az_config.DEFAULT_DOCKERFILE) self.build_runtime_docker(default_runtime_img_name, dockerfile) os.remove(dockerfile) else: raise Exception('docker command not found. Install docker or use ' 'an already built runtime') def build_runtime(self, runtime_name, requirements_file=None, extra_args=[]): try: shutil.rmtree(az_config.BUILD_DIR) except Exception: pass action_name = self._format_action_name(runtime_name) build_dir = os.path.join(az_config.BUILD_DIR, action_name) os.makedirs(build_dir, exist_ok=True) logger.info('Building default runtime in {}'.format(build_dir)) action_dir = os.path.join(build_dir, az_config.ACTION_DIR) os.makedirs(action_dir, exist_ok=True) req_file = os.path.join(build_dir, 'requirements.txt') with open(req_file, 'w') as reqf: reqf.write(az_config.REQUIREMENTS_FILE) if not is_unix_system(): if 'dev' in lithops.__version__: reqf.write('git+https://github.com/lithops-cloud/lithops') else: reqf.write('lithops=={}'.format(lithops.__version__)) host_file = os.path.join(build_dir, 'host.json') with open(host_file, 'w') as hstf: hstf.write(az_config.HOST_FILE) fn_file = os.path.join(action_dir, 'function.json') if self.invocation_type == 'event': with open(fn_file, 'w') as fnf: in_q_name = self._format_queue_name(action_name, az_config.IN_QUEUE) az_config.BINDINGS_QUEUE['bindings'][0][ 'queueName'] = in_q_name out_q_name = self._format_queue_name(action_name, az_config.OUT_QUEUE) az_config.BINDINGS_QUEUE['bindings'][1][ 'queueName'] = out_q_name fnf.write(json.dumps(az_config.BINDINGS_QUEUE)) elif self.invocation_type == 'http': with open(fn_file, 'w') as fnf: fnf.write(json.dumps(az_config.BINDINGS_HTTP)) entry_point = os.path.join(os.path.dirname(__file__), 'entry_point.py') main_file = os.path.join(action_dir, '__init__.py') shutil.copy(entry_point, main_file) if is_unix_system(): mod_dir = os.path.join(build_dir, az_config.ACTION_MODULES_DIR) os.chdir(build_dir) cmd = '{} -m pip install -U -t {} -r requirements.txt'.format( sys.executable, mod_dir) if logger.getEffectiveLevel() != logging.DEBUG: cmd = cmd + " >{} 2>&1".format(os.devnull) os.system(cmd) create_handler_zip(az_config.FH_ZIP_LOCATION, entry_point, '__init__.py') archive = zipfile.ZipFile(az_config.FH_ZIP_LOCATION) archive.extractall(path=mod_dir) os.remove(mod_dir + '/__init__.py') os.remove(az_config.FH_ZIP_LOCATION) def _create_function(self, docker_image_name, memory, timeout): """ Create and publish an Azure Functions """ action_name = self._format_action_name(docker_image_name, memory) logger.info( 'Creating new Lithops runtime for Azure Function: {}'.format( action_name)) if self.invocation_type == 'event': try: in_q_name = self._format_queue_name(action_name, az_config.IN_QUEUE) logger.debug('Creating queue {}'.format(in_q_name)) self.queue_service.create_queue(in_q_name) except Exception: in_queue = self.queue_service.get_queue_client(in_q_name) in_queue.clear_messages() try: out_q_name = self._format_queue_name(action_name, az_config.OUT_QUEUE) logger.debug('Creating queue {}'.format(out_q_name)) self.queue_service.create_queue(out_q_name) except Exception: out_queue = self.queue_service.get_queue_client(out_q_name) out_queue.clear_messages() python_version = version_str(sys.version_info) cmd = ( 'az functionapp create --name {} --storage-account {} ' '--resource-group {} --os-type Linux --runtime python ' '--runtime-version {} --functions-version {} --consumption-plan-location {}' .format(action_name, self.storage_account_name, self.resource_group, python_version, self.functions_version, self.location)) if logger.getEffectiveLevel() != logging.DEBUG: cmd = cmd + " >{} 2>&1".format(os.devnull) res = os.system(cmd) if res != 0: raise Exception( 'There was an error creating the function in Azure. cmd: {}'. format(cmd)) logger.debug('Publishing function: {}'.format(action_name)) build_dir = os.path.join(az_config.BUILD_DIR, action_name) os.chdir(build_dir) res = 1 while res != 0: time.sleep(5) if is_unix_system(): cmd = 'func azure functionapp publish {} --python --no-build'.format( action_name) else: cmd = 'func azure functionapp publish {} --python'.format( action_name) if logger.getEffectiveLevel() != logging.DEBUG: cmd = cmd + " >{} 2>&1".format(os.devnull) res = os.system(cmd) time.sleep(10) def delete_runtime(self, runtime_name, memory): """ Deletes a runtime """ action_name = self._format_action_name(runtime_name, memory) logger.debug('Deleting function app: {}'.format(action_name)) cmd = ('az functionapp delete --name {} --resource-group {}'.format( action_name, self.resource_group)) if logger.getEffectiveLevel() != logging.DEBUG: cmd = cmd + " >{} 2>&1".format(os.devnull) os.system(cmd) try: in_q_name = self._format_queue_name(action_name, az_config.IN_QUEUE) self.queue_service.delete_queue(in_q_name) except Exception: pass try: out_q_name = self._format_queue_name(action_name, az_config.OUT_QUEUE) self.queue_service.delete_queue(out_q_name) except Exception: pass def invoke(self, docker_image_name, memory=None, payload={}, return_result=False): """ Invoke function """ action_name = self._format_action_name(docker_image_name, memory) if self.invocation_type == 'event': in_q_name = self._format_queue_name(action_name, az_config.IN_QUEUE) in_queue = self.queue_service.get_queue_client(in_q_name) msg = in_queue.send_message(dict_to_b64str(payload)) activation_id = msg.id if return_result: out_q_name = self._format_queue_name(action_name, az_config.OUT_QUEUE) out_queue = self.queue_service.get_queue_client(out_q_name) msg = [] while not msg: time.sleep(1) msg = out_queue.receive_message() out_queue.clear_messages() return b64str_to_dict(msg.content) elif self.invocation_type == 'http': endpoint = "https://{}.azurewebsites.net".format(action_name) parsed_url = urlparse(endpoint) ctx = ssl._create_unverified_context() conn = http.client.HTTPSConnection(parsed_url.netloc, context=ctx) route = "/api/lithops_handler" if return_result: conn.request("GET", route, body=json.dumps(payload, default=str)) resp = conn.getresponse() data = json.loads(resp.read().decode("utf-8")) conn.close() return data else: # logger.debug('Invoking calls {}'.format(', '.join(payload['call_ids']))) conn.request("POST", route, body=json.dumps(payload, default=str)) resp = conn.getresponse() if resp.status == 429: time.sleep(0.2) conn.close() return None activation_id = resp.read().decode("utf-8") conn.close() return activation_id def get_runtime_key(self, docker_image_name, runtime_memory): """ Method that creates and returns the runtime key. Runtime keys are used to uniquely identify runtimes within the storage, in order to know which runtimes are installed and which not. """ action_name = self._format_action_name(docker_image_name, runtime_memory) runtime_key = os.path.join(self.name, action_name) return runtime_key def clean(self): """ Deletes all Lithops Azure Function Apps runtimes """ logger.debug('Deleting all runtimes') runtimes = self.list_runtimes() for runtime in runtimes: runtime_name, runtime_memory = runtime self.delete_runtime(runtime_name, runtime_memory) def _generate_runtime_meta(self, docker_image_name, memory): """ Extract installed Python modules from Azure runtime """ logger.info( "Extracting Python modules from: {}".format(docker_image_name)) payload = { 'log_level': logger.getEffectiveLevel(), 'get_preinstalls': True } try: runtime_meta = self.invoke(docker_image_name, memory=memory, payload=payload, return_result=True) except Exception: raise Exception("Unable to invoke 'extract-preinstalls' action") if not runtime_meta or 'preinstalls' not in runtime_meta: raise Exception(runtime_meta) logger.debug("Extracted metadata succesfully") return runtime_meta def list_runtimes(self, docker_image_name='all'): """ List all the Azure Function Apps deployed. return: Array of tuples (function_name, memory) """ logger.debug('Listing all functions deployed...') functions = [] response = os.popen( 'az functionapp list --query "[].defaultHostName\"').read() response = json.loads(response) for function in response: function = function.replace('.azurewebsites.net', '') if docker_image_name == function or docker_image_name == 'all': functions.append((function, '')) logger.debug('Listed {} functions'.format(len(functions))) return functions
raw_input('Press Enter to continue...') # Each storage account has a primary and secondary access key. # These keys are used by aplications to access data in your storage account, such as Queues. # Obtain the primary storage access key for use with the rest of the demo response = azurerm.get_storage_account_keys(auth_token, subscription_id, resourcegroup_name, storageaccount_name) storageaccount_keys = json.loads(response.text) storageaccount_primarykey = storageaccount_keys['keys'][0]['value'] # Create the Queue with the Azure Storage SDK and the access key obtained in the previous step queue_service = QueueServiceClient(account_name=storageaccount_name, account_key=storageaccount_primarykey) response = queue_service.create_queue('pizzaqueue') if response == True: print('Storage Queue: pizzaqueue created successfully.\n') else: print('Error creating Storage Queue.\n') ### # Use the Azure Storage Storage SDK for Python to drop some messages in our Queue ### print( 'Now let\'s drop some messages in our Queue.\nThese messages could indicate a take-out order being received for a customer ordering pizza.' ) raw_input('Press Enter to continue...') # This basic example creates a message for each pizza ordered. The message is *put* on the Queue. queue_service.put_message('pizzaqueue', u'Veggie pizza ordered.')