def get_queue_client_by_uri(queue_uri): storage = StorageUtilities.get_storage_from_uri(queue_uri) queue_service = QueueService(account_name=storage.storage_name, account_key=storage.key) queue_service.create_queue(storage.container_name) return queue_service, storage.container_name
class AzureQueue(object): def __init__(self, queue_name): self.conn = QueueService(account_name=os.getenv('AZURE_ACCOUNT_NAME'), account_key=os.getenv('AZURE_ACCOUNT_KEY')) self.queue_name = queue_name self.conn.create_queue(queue_name) self.conn.encode_function = QueueMessageFormat.binary_base64encode self.conn.decode_function = QueueMessageFormat.binary_base64decode def enqueue(self, func, *args, **kwargs): task = SimpleTask(func, *args, **kwargs) serialized_task = pickle.dumps(task, protocol=pickle.HIGHEST_PROTOCOL) self.conn.put_message(self.queue_name, serialized_task) return task.id def dequeue(self): messages = self.conn.get_messages(self.queue_name) if len(messages) == 1: serialized_task = messages[0] task = pickle.loads(serialized_task.content) self.conn.delete_message(self.queue_name, serialized_task.id, serialized_task.pop_receipt) return task def get_length(self): metadata = self.conn.get_queue_metadata(self.queue_name) return metadata.approximate_message_count
def get_queue_client_by_uri(queue_uri, session=None): storage = StorageUtilities.get_storage_from_uri(queue_uri, session) queue_service = QueueService(account_name=storage.storage_name, account_key=storage.key) queue_service.create_queue(storage.container_name) return queue_service, storage.container_name
def get_queue_client_by_uri(queue_uri): queue_name, storage_name, key = StorageUtilities.get_storage_from_uri(queue_uri) queue_service = QueueService(account_name=storage_name, account_key=key) queue_service.create_queue(queue_name) return queue_service, queue_name
def get_queue_client_by_uri(queue_uri, session): storage = StorageUtilities.get_storage_from_uri(queue_uri, session) queue_service = QueueService(account_name=storage.storage_name, token_credential=storage.token) queue_service.create_queue(storage.container_name) return queue_service, storage.container_name
def CreateQueue(azureQueueAccountName, azureQueueKey, azureQueueAnalysisResults): queue_service = QueueService(account_name=azureQueueAccountName, account_key=azureQueueKey) #create queue if doesnt exist if not queue_service.exists(azureQueueAnalysisResults): queue_service.create_queue(azureQueueAnalysisResults) return queue_service
class QueueStorageHandler(logging.Handler): """ Handler class which sends log messages to a Azure Storage queue. """ def __init__(self, account_name=None, account_key=None, protocol='https', queue='logs', message_ttl=None, visibility_timeout=None, base64_encoding=False, is_emulated=False, ): """ Initialize the handler. """ logging.Handler.__init__(self) self.service = QueueService(account_name=account_name, account_key=account_key, is_emulated=is_emulated, protocol=protocol) self.meta = {'hostname': gethostname(), 'process': os.getpid()} self.queue = _formatName(queue, self.meta) self.queue_created = False self.message_ttl = message_ttl self.visibility_timeout = visibility_timeout self.base64_encoding = base64_encoding def emit(self, record): """ Emit a record. Format the record and send it to the specified queue. """ try: if not self.queue_created: self.service.create_queue(self.queue) self.queue_created = True record.hostname = self.meta['hostname'] msg = self._encode_text(self.format(record)) self.service.put_message(self.queue, msg, self.visibility_timeout, self.message_ttl) except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record) def _encode_text(self, text): if self.base64_encoding: text = b64encode(text.encode('utf-8')).decode('ascii') # fallback for the breaking change in azure-storage 0.33 elif sys.version_info < (3,): if not isinstance(text, unicode): text = text.decode('utf-8') return text
def check(): queue_service = QueueService(account_name=account_name, account_key=account_key) queue_service.create_queue('monitoring') while True: print("Adding new user to scrap...") queue_service.put_message('monitoring', f'monitoring_{int(time.time())}') time.sleep(10)
def get_queue_client_by_uri(queue_uri): queue_name, storage_name, key = StorageUtilities.get_storage_from_uri( queue_uri) queue_service = QueueService(account_name=storage_name, account_key=key) queue_service.create_queue(queue_name) return queue_service, queue_name
class QueueStorageHandler(logging.Handler): """ Handler class which sends log messages to a Azure Storage queue. """ def __init__( self, account_name=None, account_key=None, protocol='https', queue='logs', message_ttl=None, visibility_timeout=None, base64_encoding=False, is_emulated=False, ): """ Initialize the handler. """ logging.Handler.__init__(self) self.service = QueueService(account_name=account_name, account_key=account_key, is_emulated=is_emulated, protocol=protocol) self.meta = {'hostname': gethostname(), 'process': os.getpid()} self.queue = _formatName(queue, self.meta) self.queue_created = False self.message_ttl = message_ttl self.visibility_timeout = visibility_timeout self.base64_encoding = base64_encoding def emit(self, record): """ Emit a record. Format the record and send it to the specified queue. """ try: if not self.queue_created: self.service.create_queue(self.queue) self.queue_created = True record.hostname = self.meta['hostname'] msg = self._encode_text(self.format(record)) self.service.put_message(self.queue, msg, self.visibility_timeout, self.message_ttl) except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record) def _encode_text(self, text): if self.base64_encoding: text = b64encode(text.encode('utf-8')).decode('ascii') # fallback for the breaking change in azure-storage 0.33 elif sys.version_info < (3, ): if not isinstance(text, unicode): text = text.decode('utf-8') return text
def get_queue_client_by_uri(queue_uri, session): storage = StorageUtilities.get_storage_from_uri(queue_uri, session) queue_service = QueueService( account_name=storage.storage_name, token_credential=storage.token) queue_service.create_queue(storage.container_name) return queue_service, storage.container_name
def create_storage_queue(self, resource_properties): storage_client = StorageManagementClient(self.credentials, self.subscription_id) key = storage_client.storage_accounts.list_keys(resource_properties['ResourceGroupName'], resource_properties['AccountName']).keys[0].value queue_service = QueueService(account_name=resource_properties['AccountName'], account_key=key) queue_service.create_queue(resource_properties['Name']) return { 'Name': resource_properties['Name'] }
class SimulationListener(Thread): def __init__(self): Thread.__init__(self) self._quit = Event() self.daemon = True self.log = logging.getLogger(__name__) settings = Settings() self.create_queue = 'create-simulated-machine' self.destroy_queue = 'destroy-simulated-machine' self.queue_service = QueueService( account_name=settings.azure_queue_account, account_key=settings.azure_queue_key ) self.queue_service.create_queue(self.create_queue) self.queue_service.create_queue(self.destroy_queue) def run(self): # dislike of unstoppable threads while not self._quit.is_set(): try: messages = self.queue_service.get_messages(self.create_queue, numofmessages=10) for m in messages: machine_json = b64decode(m.message_text) machine = json.loads(machine_json) self.log.info("Creating: " + machine["Name"] + " on " + machine["Provider"]) self.log.debug(machine_json) requests.post("http://localhost:8080/machines", machine_json) self.queue_service.delete_message(self.create_queue, m.message_id, m.pop_receipt) except Exception as e: self.log.error(e.message) try: messages = self.queue_service.get_messages(self.destroy_queue, numofmessages=10) for m in messages: machine_json = b64decode(m.message_text) machine = json.loads(machine_json) self.log.info("Deleting: " + machine["Name"] + " on " + machine["Provider"]) self.log.debug(machine_json) requests.delete("http://localhost:8080/machines/" + machine["MachineId"]) self.queue_service.delete_message(self.destroy_queue, m.message_id, m.pop_receipt) except Exception as e: self.log.error(e.message) sleep(1) def quit(self): self._quit.set()
def create_queue_from_storage_account(storage_account, name, session): token = StorageUtilities.get_storage_token(session) queue_service = QueueService( account_name=storage_account.name, token_credential=token) return queue_service.create_queue(name)
class AzureQueue(object): def __init__(self, account_name, account_key, queue_name): self.queue_name = queue_name self.queue_service = QueueService(account_name=account_name, account_key=account_key) self.queue_service.create_queue(self.queue_name) def put_message_into_queue(self, content) -> QueueMessage: """ Publishes a message with `content` :param content: The queue message :returns: A QueueMessage that has the message as well as metadata :rtype: QueueMessage """ return self.queue_service.put_message(self.queue_name, content) def get_messages(self) -> list: """ Retrieves all of the messages that have been published into queue :param content: The queue message :returns: List of Queue messages :rtype: list """ return self.queue_service.get_messages(self.queue_name) def delete_message_from_queue(self, message_id, pop_receipt): self.queue_service.delete_message(self.queue_name, message_id, pop_receipt) def get_message_count(self): queue_metadata = self.queue_service.get_queue_metadata(self.queue_name) return queue_metadata.approximate_message_count def delete(self): return self.queue_service.delete_queue(self.queue_name) def empty(self): messages = queue_service.get_messages(self.queue_name, num_messages=BATCH_NUMBER, visibility_timeout=TIMEOUT_IN_SECONDS) for message in messages: self.queue_service.delete_message(self.queue_name, message.id, message.pop_receipt)
def __createstorage(): global container_name global queue_service global block_blob_service global queue_service block_blob_service = BlockBlobService( account_name=STORAGE_ACCOUNT_NAME, account_key=STORAGE_ACCOUNT_KEY, endpoint_suffix=STORAGE_ACCOUNT_SUFFIX) timestr = time.strftime("%Y%m%d-%H%M%S") container_name = 'fromcamera' + timestr block_blob_service.create_container(container_name) block_blob_service.set_container_acl(container_name, public_access=PublicAccess.Container) queue_service = QueueService(account_name=STORAGE_ACCOUNT_NAME, account_key=STORAGE_ACCOUNT_KEY, endpoint_suffix=STORAGE_ACCOUNT_SUFFIX) queue_service.create_queue('fromcamera' + timestr)
class WorkloadTracker(object): """ Dedicated class to track important events during the running of the workload. """ def __init__(self, logger): self.config = Config() self.logger = logger self.init_storage_services() def init_storage_services(self): """ Initializes the storage service clients using values from config.py. :return: True on success. False on failure. :rtype: boolean """ try: # creates instances of Azure QueueService self.workload_queue_service = QueueService( account_name=self.config.storage_account_name, sas_token=self.config.workload_tracker_sas_token) self.workload_queue_service.create_queue( self.config.workload_tracker_queue_name) self.workload_queue_service.encode_function = models.QueueMessageFormat.noencode return True except Exception as ex: self.logger.Exception(ex, self.init_storage_services.__name__) return False def write(self, event_type, content=None): """ Write the event to the dedicated workload tracker queue """ # create an event evt = WorkloadEvent() evt.event_type = int(event_type) evt.content = content # write serialized event to Azure queue serialized_event = json.dumps(evt.__dict__) self.workload_queue_service.put_message( self.config.workload_tracker_queue_name, serialized_event)
def queue(): account_name = config.STORAGE_ACCOUNT_NAME account_key = config.STORAGE_ACCOUNT_KEY queue_service = QueueService(account_name=account_name, account_key=account_key) print("Creating task queue") task_queue_name = config.TASK_QUEUE_NAME queue_service.create_queue(task_queue_name) print("Task queue created") queue_service.put_message(task_queue_name, u'message1') messages = queue_service.get_messages(task_queue_name, num_messages=16) for message in messages: print(message.content) queue_service.delete_message(task_queue_name, message.id, message.pop_receipt)
def send_tweet(tweet_message, in_reply_to, entity): queue_name = settings.AZURE_QUEUE_NAME queue_service = QueueService( account_name=settings.TWITTERBOT_STORAGE_ACCOUNT_NAME, account_key=settings.TWITTERBOT_STORAGE_ACCOUNT_KEY) queue_service.create_queue(queue_name) queue_message = { 'id': entity['id'], 'tweet': { 'status': tweet_message, 'in_reply_to_status_id': in_reply_to }, 'percentiles': entity['percentiles'] } queue_service.put_message( queue_name, base64.b64encode( json.dumps(queue_message).encode('utf-8')).decode('utf-8'))
class MoeHandler(): def __init__(self, api, generator, queue_name, conn_str): self.api = api self.gen_util = Generate_util(generator) self.queue_name = queue_name self.queue_service = QueueService(connection_string=conn_str) self.queue_service.create_queue(queue_name) # loops forever, reading messages and sending to gen util for parsing. Then sends the message def read_messages(self): while 1: try: messages = self.queue_service.get_messages(self.queue_name) if not messages: time.sleep(2) # We'll wait a few seconds and check again. continue for message in messages: event = json.loads(message.content) if event['msg_type'] == 'tweet': self.handle_tweet(event) elif event['msg_type'] == 'dm': self.handle_dm(event) self.queue_service.delete_message( self.queue_name, message.id, message.pop_receipt) except Exception as e: print("Exception handling messages") print(e) def handle_dm(self, event): imgpth = self.gen_util.parse_msg(event['text']) media = self.api.media_upload(imgpth) self.api.send_direct_message( recipient_id=event['user_id'], text="Here's your generated anime girl!", attachment_type='media', attachment_media_id=media.media_id) return def handle_tweet(self, event): imgpth = self.gen_util.parse_msg(event['text']) self.api.update_with_media(imgpth, status="Here's your generated anime girl!", in_reply_to_status_id=event['respond_id'], auto_populate_reply_metadata=True) return
def run_temperature(): queue_service = QueueService(account_name, account_key) queue1 = "NormalTemp" queue2 = "HighTemp" queue_service.create_queue(queue1) print("Queue created successfully...") queue_service.create_queue(queue2) print("Queue created successfully...") for i in range(1, 10): temp = generateTemp() queue_service.put_message(queue1, temp) if(temp > 80): # Code for alert function will be plcaed here tkMessageBox.showinfo(message=temp"is the temperature") queue_service.put_message(queue2, temp)
class AzureStorageQueue(Common.Contracts.Queue): def __init__(self, queue_name, config: AzureStorageConfig): self._queue_name = queue_name self._queue_service = QueueService(account_name=config.account_name, account_key=config.account_key) self._queue_service.encode_function = QueueMessageFormat.text_base64encode self._queue_service.decode_function = QueueMessageFormat.text_base64decode def push(self, message): self._queue_service.create_queue(self._queue_name) self._queue_service.put_message(self._queue_name, message) def pop(self): self._queue_service.create_queue(self._queue_name) messages = self._queue_service.peek_messages(self._queue_name) for message in messages: result = message.content self._queue_service.delete_message(self._queue_name, message.id, message.pop_receipt) return result def peek(self): self._queue_service.create_queue(self._queue_name) messages = self._queue_service.peek_messages(self._queue_name) for message in messages: return message.content
def run(job, **kwargs): resource = kwargs.get("resource") env_id = "{{ env_id }}" env = Environment.objects.get(id=env_id) rh = env.resource_handler.cast() resource_group = "{{ resource_group }}" create_custom_fields_as_needed() storage_account = "{{ storage_account }}" azure_queue_name = "{{ azure_queue_name }}" set_progress("Connecting To Azure Management Service...") azure_client = get_azure_storage_client(rh) res = azure_client.storage_accounts.list_keys(resource_group, storage_account) keys = res.keys set_progress("Connecting To Azure queues...") queue_service = QueueService(account_name="{{ storage_account }}", account_key=keys[0].value) set_progress("Creating a file...") if queue_service.exists(queue_name=azure_queue_name): return ( "FAILURE", "Queue with this name already exists", "The queue can not be created.", ) else: queue_service.create_queue("{{ azure_queue_name }}") resource.name = azure_queue_name resource.azure_storage_account_name = storage_account resource.azure_account_key = keys[0].value resource.azure_account_key_fallback = keys[1].value resource.azure_storage_queue_name = "Azure queues - " + azure_queue_name resource.save() return "Success", "", ""
def set_up_queue(queue_name): account_name = input('Please provide Azure account name:') account_key = input('Please provide Azure access key:') queue_service = QueueService(account_name, account_key) print('Creating queue: {} ...'.format(queue_name)) try: queue_service.create_queue(queue_name) except AzureException as exc: print('Cannot create queue, reason: {}'.format(exc), file=sys.stderr) sys.exit(1) else: print('Queue successfully created.') print('Generating SAS token...') expiry_date = datetime.date.today() + datetime.timedelta(days=30) print('SAS token will expire in', str(expiry_date)) access_policy = AccessPolicy( expiry=str(expiry_date), permission=( QueueSharedAccessPermissions.READ + QueueSharedAccessPermissions.ADD + QueueSharedAccessPermissions.PROCESS ), ) sas_token = queue_service.generate_shared_access_signature( queue_name, SharedAccessPolicy(access_policy), ) print('Generated SAS token:', sas_token) print('Please use above token in QueueManager.')
def GetQueue(): u = keys.Key().generate_keys() keys.Key().save_key(DocumentManagement.randomString(), u['privateKey']) #print(str(u['publicKey'], 'utf-8')) queue_service = QueueService( account_name=cfg.settings['STORAGE_ACCOUNT'], account_key=cfg.settings['STORAGE_ACCOUNT_KEY']) queueName = DocumentManagement.randomString() created = queue_service.create_queue(queueName) data = None if created == True: data = {'key': str(u['publicKey'], 'utf-8'), 'queue': queueName} return data
class AzureStorageQueue(Queue): """Interface for interacting with an Azure Storage Queue (through the Queue contract)""" def __init__(self, queue_name, config: AzureStorageConfig): """Initializes the storage queue. :param queue_name: The name of the queue to access. If a queue with this name doesn't already exist on the storage account, the queue will be created on the first operation. :param config: AzureStorageConfig with a valid account name and account key """ self._queue_name = queue_name self._queue_service = QueueService(account_name=config.account_name, account_key=config.account_key) self._queue_service.encode_function = \ QueueMessageFormat.text_base64encode self._queue_service.decode_function = \ QueueMessageFormat.text_base64decode def push(self, message): """Pushes a new message onto the queue.""" self._queue_service.create_queue(self._queue_name) self._queue_service.put_message(self._queue_name, message) def pop(self): """Pops the first message from the queue and returns it.""" self._queue_service.create_queue(self._queue_name) # get_messages prevents another client from getting the message # before we've had a chance to delete it. The visibility_timeout # prevents the message from being seen by other clients # for X seconds. messages = self._queue_service.get_messages(self._queue_name, visibility_timeout=30) for message in messages: result = message.content self._queue_service.delete_message(self._queue_name, message.id, message.pop_receipt) return result def peek(self): """Peeks the fist message from the queue and returns it.""" self._queue_service.create_queue(self._queue_name) messages = self._queue_service.peek_messages(self._queue_name) for message in messages: return message.content @staticmethod def create(queue_name: str): """Helper function for creating a Azure Storage Queue from the storage_config property defined inside of AzureConfig.""" azure_config = AzureConfig() return AzureStorageQueue(queue_name, azure_config.storage_config)
class AzureFunctionAppBackend: """ A wrap-up around Azure Function Apps backend. """ def __init__(self, config): self.log_level = os.getenv('CLOUDBUTTON_LOGLEVEL') self.name = 'azure_fa' self.config = config self.fa_client = FunctionAppClient(self.config) self.queue_service = QueueService( account_name=self.config['account_name'], account_key=self.config['account_key']) self.queue_service.encode_function = QueueMessageFormat.text_base64encode self.queue_service.decode_function = QueueMessageFormat.text_base64decode log_msg = 'Cloudbutton v{} init for Azure Function Apps'.format( __version__) logger.info(log_msg) if not self.log_level: print(log_msg) def create_runtime(self, docker_image_name, memory=None, timeout=azure_fa_config.RUNTIME_TIMEOUT_DEFAULT): """ Creates a new runtime into Azure Function Apps from the provided Linux image for consumption plan """ log_msg = 'Creating new Cloudbutton runtime for Azure Function Apps...' logger.info(log_msg) if not self.log_level: print(log_msg) logger.info('Extracting preinstalls for Azure runtime') metadata = self._generate_runtime_meta() logger.info('Creating new Cloudbutton runtime') action_name = self._format_action_name(docker_image_name) self._create_runtime(action_name) return metadata def delete_runtime(self, docker_image_name, extract_preinstalls=False): """ Deletes a runtime """ if extract_preinstalls: action_name = docker_image_name else: action_name = self._format_action_name(docker_image_name) self.fa_client.delete_action(action_name) queue_name = self._format_queue_name(docker_image_name, type='trigger') self.queue_service.delete_queue(queue_name) def invoke(self, docker_image_name, memory=None, payload={}): """ Invoke function """ action_name = self._format_action_name(docker_image_name) queue_name = self._format_queue_name(action_name, type='trigger') try: msg = self.queue_service.put_message(queue_name, json.dumps(payload)) activation_id = msg.id except Exception: logger.debug('Creating queue (invoke)') self.queue_service.create_queue(queue_name) return self.invoke(docker_image_name, memory=memory, payload=payload) return activation_id def get_runtime_key(self, docker_image_name, runtime_memory): """ Method that creates and returns the runtime key. Runtime keys are used to uniquely identify runtimes within the storage, in order to know which runtimes are installed and which not. """ action_name = self._format_action_name(docker_image_name) runtime_key = os.path.join(self.name, action_name) return runtime_key def _format_action_name(self, action_name): sha_1 = hashlib.sha1() block = action_name.encode('ascii', errors='ignore') sha_1.update(block) tag = sha_1.hexdigest()[:8] sha_1 = hashlib.sha1() block = self.config['account_name'].encode('ascii', errors='ignore') sha_1.update(block) tag = tag + sha_1.hexdigest()[:8] version = re.sub(r'[/_:.-]', '', __version__) action_name = action_name[:16] + '-' + version[:5] + '-' + tag return action_name def _format_queue_name(self, action_name, type): # Using different queue names because there is a delay between # deleting a queue and creating another one with the same name return action_name + '-' + type def _create_runtime(self, action_name, extract_preinstalls=False): """ Creates a new runtime with the base modules and cloudbutton """ def add_base_modules(): cmd = 'pip3 install -t {} -r requirements.txt'.format( azure_fa_config.ACTION_MODULES_DIR) child = sp.Popen(cmd, shell=True, stdout=sp.PIPE, stderr=sp.PIPE) # silent child.wait() logger.debug(child.stdout.read().decode()) logger.debug(child.stderr.read().decode()) if child.returncode != 0: cmd = 'pip install -t {} -r requirements.txt'.format( azure_fa_config.ACTION_MODULES_DIR) child = sp.Popen(cmd, shell=True, stdout=sp.PIPE, stderr=sp.PIPE) # silent child.wait() logger.debug(child.stdout.read().decode()) logger.debug(child.stderr.read().decode()) if child.returncode != 0: logger.critical( 'Failed to install base modules for Azure Function') exit(1) def add_cloudbutton_module(): module_location = os.path.dirname( os.path.abspath(cloudbutton.__file__)) shutil.copytree( module_location, os.path.join(azure_fa_config.ACTION_MODULES_DIR, 'cloudbutton')) def get_bindings_str(action_name, extract_preinstalls=False): if not extract_preinstalls: bindings = { "scriptFile": "__init__.py", "bindings": [{ "name": "msgIn", "type": "queueTrigger", "direction": "in", "queueName": self._format_queue_name(action_name, 'trigger'), "connection": "AzureWebJobsStorage" }] } else: bindings = { "scriptFile": "__init__.py", "bindings": [{ "name": "msgIn", "type": "queueTrigger", "direction": "in", "queueName": self._format_queue_name(action_name, type='trigger'), "connection": "AzureWebJobsStorage" }, { "name": "msgOut", "type": "queue", "direction": "out", "queueName": self._format_queue_name(action_name, type='result'), "connection": "AzureWebJobsStorage" }] } return json.dumps(bindings) initial_dir = os.getcwd() temp_folder = next(tempfile._get_candidate_names()) os.mkdir(temp_folder) os.chdir(temp_folder) try: # Create project folder from template project_template = os.path.join( os.path.dirname(os.path.abspath(__file__)), 'action') project_dir = os.path.join(initial_dir, temp_folder, action_name) shutil.copytree(project_template, project_dir) os.chdir(project_dir) action_dir = os.path.join(project_dir, action_name) os.rename('action', action_dir) # Add the base dependencies and current cloudbutton module logger.debug('Adding runtime base modules') os.makedirs(azure_fa_config.ACTION_MODULES_DIR, exist_ok=True) add_base_modules() add_cloudbutton_module() # Set entry point file if extract_preinstalls: entry_point_file = 'extract_preinstalls_action.py' else: entry_point_file = 'handler_action.py' os.rename(os.path.join(action_dir, entry_point_file), os.path.join(action_dir, '__init__.py')) # Edit the function's bindings for it to be a queue triggered function with open(os.path.join(action_dir, 'function.json'), 'w') as bindings_file: bindings_file.write( get_bindings_str(action_name, extract_preinstalls)) # Create trigger queue, create action logger.debug('Creating trigger queue') queue_name = self._format_queue_name(action_name, type='trigger') self.queue_service.create_queue(queue_name) self.fa_client.create_action(action_name) except Exception as e: raise Exception("Unable to create the new runtime", e) finally: os.chdir(initial_dir) shutil.rmtree(temp_folder, ignore_errors=True) # Remove tmp project folder def _generate_runtime_meta(self): """ Extract installed Python modules from Azure runtime """ action_name = 'cloudbutton-extract-preinstalls-' + get_unique_id() self._create_runtime(action_name, extract_preinstalls=True) logger.debug("Invoking 'extract-preinstalls' action") try: runtime_meta = self._invoke_with_result(action_name) except Exception: raise Exception("Unable to invoke 'extract-preinstalls' action") try: self.delete_runtime(action_name, extract_preinstalls=True) except Exception: raise Exception("Unable to delete 'extract-preinstalls' action") if not runtime_meta or 'preinstalls' not in runtime_meta: raise Exception(runtime_meta) logger.debug("Extracted metadata succesfully") return runtime_meta def _invoke_with_result(self, action_name): result_queue_name = self._format_queue_name(action_name, type='result') self.queue_service.create_queue(result_queue_name) trigger_queue_name = self._format_queue_name(action_name, type='trigger') self.queue_service.put_message(trigger_queue_name, '') msg = [] while not msg: msg = self.queue_service.get_messages(result_queue_name, num_messages=1) time.sleep(0.5) result_str = msg[0].content self.queue_service.delete_queue(result_queue_name) return json.loads(result_str)
def put_to_queue(name, content): queue_service = QueueService(account_name=config.AZURE_STORAGE_NAME, account_key=config.AZURE_STORAGE_KEY) queue_service.create_queue(name) queue_service.put_message(name, content)
class AzureService(object): VISIBILITY_TIMEOUT = 5*60 def __init__(self, connection_string, container_name, queue_get, queue_push, logger=None): self.ctnname = container_name self.getname = queue_get self.pushname = queue_push self.qs = QueueService(connection_string=connection_string, protocol='https', # endpoint_suffix='core.windows.net' ) self.bs = BlockBlobService(connection_string=connection_string) self.qs.create_queue(self.getname, timeout=1) self.qs.create_queue(self.pushname, timeout=1) self.bs.create_container(self.ctnname, timeout=1) if logger: logger.info('Init Azure success') def pushMessage(self, message, qname=None, logger=None): if qname is None: qname = self.pushname try: self.qs.put_message(self.pushname, message) except Exception as e: if logger: logger.exception('ERROR PUSH MESSAGE ') else: print 'ERROR PUSH MESSAGE ' print e def getMessage(self, qname=None, num=1, logger=None): if qname is None: qname = self.getname try: message = self.qs.get_messages(qname, num, visibility_timeout=self.VISIBILITY_TIMEOUT) except Exception as e: if logger: logger.exception('ERROR GET MESSAGE ') else: print 'ERROR GET MESSAGE ' print e return [] return message def getReceiptInfo(self, logger=None): message = self.getMessage(logger=logger) if len(message) > 0: rinfo = ReceiptSerialize.fromjson(message[0].content) return message[0], rinfo else: return None, None def count(self): metadata_get = self.qs.get_queue_metadata(self.getname) metadata_push = self.qs.get_queue_metadata(self.pushname) generator = self.bs.list_blobs(self.ctnname) bc = 0 for blob in generator: bc += 1 return {'get_count' : metadata_get.approximate_message_count, 'push_count': metadata_push.approximate_message_count, 'blob_count': bc } def uploadFolder(self, folderpath, logger): for filename in os.listdir(folderpath): if len(filename) > 4: suffix = filename[-4:].upper() else: continue if '.JPG' == suffix or 'JPEG' == suffix: receipt_metadata = ReceiptSerialize() receipt_metadata.receiptBlobName = unicode(filename, 'utf-8') self.qs.put_message(self.getname, b64encode(receipt_metadata.toString()).decode('utf-8')) self.bs.create_blob_from_path(self.ctnname, receipt_metadata.receiptBlobName, os.path.join(folderpath, filename), max_connections=2, timeout=None) logger.info('upload %s', filename) def getImage(self, imgname, logger=None): localpath= os.path.join(args.download_dir, imgname) try: self.bs.get_blob_to_path(self.ctnname, imgname, localpath) except AzureMissingResourceHttpError as e: if logger: logger.error('Blob named ' + imgname + ' doesnot exist.' , exc_info=True) else: print 'Blob named ' + imgname + ' doesnot exist.' print e return '' except Exception as e: if logger: logger.error('Exception while getting blob.', exc_info=True) else: print 'Exception while getting blob.' print e return None return localpath def deleteMessage(self, message, qname=None, logger=None): if qname is None: qname = self.getname try: self.qs.delete_message(qname, message.id, message.pop_receipt) except Exception as e: if logger: logger.exception('ERROR DELETE MESSAGE ') else: print 'ERROR DELETE MESSAGE ' print e def deleteImage(self, imgname, logger=None): try: self.bs.delete_blob(self.ctnname, imgname) except Exception as e: if logger: logger.exception('ERROR DELETE IMAGE ') else: print 'ERROR DELETE IMAGE ' print e def cleanUp(self): count = 0 print('deleted: ') while True: messages = self.qs.get_messages(self.getname) for message in messages: count += 1 self.qs.delete_message(self.getname, message.id, message.pop_receipt) if len(messages) == 0: break print(str(count) + ' from queue-get') count = 0 while True: messages = self.qs.get_messages(self.pushname) for message in messages: count += 1 self.qs.delete_message(self.pushname, message.id, message.pop_receipt) if len(messages) == 0: break print(str(count) + ' from queue-push') count = 0 generator = self.bs.list_blobs(self.ctnname) for blob in generator: count += 1 self.bs.delete_blob(self.ctnname, blob.name) print(str(count) + ' from container')
class StorageQueueContext(): """Initializes the repository with the specified settings dict. Required settings in config dict are: - AZURE_STORAGE_NAME - AZURE_STORAGE_KEY """ _models = [] _service = None _storage_key = '' _storage_name = '' def __init__(self, **kwargs): self._storage_name = kwargs.get('AZURE_STORAGE_NAME', '') self._storage_key = kwargs.get('AZURE_STORAGE_KEY', '') """ service init """ if self._storage_key != '' and self._storage_name != '': self._service = QueueService(account_name = self._storage_name, account_key = self._storage_key, protocol='https') """ registered models """ self._models = [] """ encrypt queue service """ if kwargs.get('AZURE_REQUIRE_ENCRYPTION', False): # Create the KEK used for encryption. # KeyWrapper is the provided sample implementation, but the user may use their own object as long as it implements the interface above. kek = KeyWrapper(kwargs.get('AZURE_KEY_IDENTIFIER', 'otrrentapi'), kwargs.get('SECRET_KEY', 'super-duper-secret')) # Key identifier # Create the key resolver used for decryption. # KeyResolver is the provided sample implementation, but the user may use whatever implementation they choose so long as the function set on the service object behaves appropriately. key_resolver = KeyResolver() key_resolver.put_key(kek) # Set the require Encryption, KEK and key resolver on the service object. self._service.require_encryption = True self._service.key_encryption_key = kek self._service.key_resolver_funcion = key_resolver.resolve_key pass def __create__(self, queue) -> bool: if (not self._service is None): try: self._service.create_queue(queue) return True except AzureException as e: log.error('failed to create {} with error {}'.format(queue, e)) return False else: return True pass def register_model(self, storagemodel:object): modelname = storagemodel.__class__.__name__ if isinstance(storagemodel, StorageQueueModel): if (not modelname in self._models): self.__create__(storagemodel._queuename) self._models.append(modelname) log.info('model {} registered successfully. Models are {!s}'.format(modelname, self._models)) pass def put(self, storagemodel:object) -> StorageQueueModel: """ insert queue message into storage """ modelname = storagemodel.__class__.__name__ if isinstance(storagemodel, StorageQueueModel): if (modelname in self._models): """ peek first message in queue """ try: message = self._service.put_message(storagemodel._queuename, storagemodel.getmessage()) storagemodel.mergemessage(message) except AzureException as e: log.error('can not save queue message: queue {} with message {} because {!s}'.format(storagemodel._queuename, storagemodel.content, e)) storagemodel = None else: log.info('please register model {} first'.format(modelname)) storagemodel = None else: log.info('model {} is not a Queue Model'.format(modelname)) storagemodel = None return storagemodel def peek(self, storagemodel:object) -> StorageQueueModel: """ lookup the next message in queue """ modelname = storagemodel.__class__.__name__ if isinstance(storagemodel, StorageQueueModel): if (modelname in self._models): """ peek first message in queue """ try: messages = self._service.peek_messages(storagemodel._queuename, num_messages=1) """ parse retrieved message """ for message in messages: storagemodel.mergemessage(message) """ no message retrieved ?""" if storagemodel.id is None: storagemodel = None except AzureException as e: log.error('can not peek queue message: queue {} with message {} because {!s}'.format(storagemodel._queuename, storagemodel.content, e)) storagemodel = None else: log.info('please register model {} first'.format(modelname)) storagemodel = None else: log.info('model {} is not a Queue Model'.format(modelname)) storagemodel = None return storagemodel def get(self, storagemodel:object, hide = 0) -> StorageQueueModel: """ lookup the next message in queue """ modelname = storagemodel.__class__.__name__ if isinstance(storagemodel, StorageQueueModel): if (modelname in self._models): """ get first message in queue """ try: if hide > 0: messages = self._service.get_messages(storagemodel._queuename, num_messages=1, visibility_timeout = hide) else: messages = self._service.get_messages(storagemodel._queuename, num_messages=1) """ parse retrieved message """ for message in messages: storagemodel.mergemessage(message) """ no message retrieved ?""" if storagemodel.id is None: storagemodel = None except AzureException as e: log.error('can not get queue message: queue {} with message {} because {!s}'.format(storagemodel._queuename, storagemodel.content, e)) storagemodel = None else: log.info('please register model {} first'.format(modelname)) storagemodel = None else: log.info('model {} is not a Queue Model'.format(modelname)) storagemodel = None return storagemodel def update(self, storagemodel:object, hide = 0) -> StorageQueueModel: """ update the message in queue """ modelname = storagemodel.__class__.__name__ if isinstance(storagemodel, StorageQueueModel): if (modelname in self._models): """ check if message in queue """ if (storagemodel.id != '') and (storagemodel.pop_receipt != '') and (not storagemodel.id is None) and (not storagemodel.pop_receipt is None): try: content = storagemodel.getmessage() message = self._service.update_message(storagemodel._queuename, storagemodel.id, storagemodel.pop_receipt, visibility_timeout = hide, content=content) storagemodel.content = content storagemodel.pop_receipt = message.pop_receipt except AzureException as e: log.error('can not update queue message: queue {} with message.id {!s} because {!s}'.format(storagemodel._queuename, storagemodel.id, e)) storagemodel = None else: log.info('cant update queuemessage {} due to missing id and pop_receipt'.format(modelname)) storagemodel = None else: log.info('please register model {} first'.format(modelname)) storagemodel = None else: log.info('model {} is not a Queue Model'.format(modelname)) storagemodel = None return storagemodel def delete(self, storagemodel:object) -> bool: """ delete the message in queue """ modelname = storagemodel.__class__.__name__ deleted = False if isinstance(storagemodel, StorageQueueModel): if (modelname in self._models): """ check if message in queue """ if (storagemodel.id != '') and (storagemodel.pop_receipt != '') and (not storagemodel.id is None) and (not storagemodel.pop_receipt is None): try: self._service.delete_message(storagemodel._queuename, storagemodel.id, storagemodel.pop_receipt) deleted = True except AzureException as e: log.error('can not delete queue message: queue {} with message.id {!s} because {!s}'.format(storagemodel._queuename, storagemodel.id, e)) else: log.info('cant update queuemessage {} due to missing id and pop_receipt'.format(modelname)) else: log.info('please register model {} first'.format(modelname)) else: log.info('model {} is not a Queue Model'.format(modelname)) return deleted
class QueueLogger(object): """ This class contains functionality to log stdout to Azure Queue Storage """ def __init__(self): """ Initializes a new instance of the QueueLogger class. :param int batch_size: The number of messages to write into a single Azure Storage Queue message. """ self.config = Config() self.queue_service = QueueService(account_name = self.config.storage_account_name, sas_token = self.config.logger_queue_sas) self.queue_service.encode_function = models.QueueMessageFormat.noencode self.init_storage() def start_listening(self): for line in fileinput.input(): self.write(line.strip()) def init_storage(self): """ Initializes storage table & queue, creating it if it doesn't exist. :return: True on succeess. False on failure. :rtype: boolean """ try: # will create the logger queue if it doesn't exist self.queue_service.create_queue(self.config.logger_queue_name) return True except Exception as ex: self.write_stdout(ex) return False def flush(self): """ Flush the internal buffer to Storage Queue """ self.put_message_to_queue() def write(self, content): """ Buffers string content to be written to Storage Queue :param str content: The content to write/buffer """ self.write_stdout(content) self.write_queue(content) def write_queue(self, content): """ Buffers string content to be written to Storage Queue :param str content: The content to write/buffer """ self.queue_service.put_message(self.config.logger_queue_name, content) def write_stdout(self, content): """ Buffers string content to be written to Storage Queue :param str content: The content to write/buffer """ print(content)
from azure.storage.queue import QueueService quename = "mytestqueue" message = [u"Hello, World!", u"Hello, House!", u"Hello, God!"] accountname = "xxxx" accountkey = "xxxx" queue_service = QueueService(account_name=accountname, account_key=accountkey) queue_service.create_queue(quename) for x in message: queue_service.put_message(quename,x) messages = queue_service.peek_messages(quename) for message in messages: print(message.content) metadata = queue_service.get_queue_metadata(quename) count = metadata.approximate_message_count print(count)
class Queue: def __init__(self, account_name, account_key, queue_name="logqueue"): """Initialiaze a queue. The type is set by the 'ACS_LOGGING_QUEUE_TYPE' environment variable. If it is set to 'AzureStorageQueue' then values must be provided for 'account_name' and 'account_key' which are values associated with the Azure Storage account. 'queue_name' is optional and defaults to 'logqueue'. """ self.log = Log() self.queue_type = config.ACS_LOGGING_QUEUE_TYPE self.queue_name = queue_name # self.log.debug("Queue type: " + self.queue_type + " / " + self.queue_name) if self.queue_type == "AzureStorageQueue": self.createAzureQueues(account_name, account_key) elif self.queue_type == "LocalFile": self.file_queue = open(config.UNPROCESSED_LOG_FILE, "w+") else: self.log.error("Unknown queue type: " + queue_type) def createAzureQueues(self, account_name, account_key): """ Create a queue for unprocessed log messages. Entries in the queue will be dequeued, processed and deleted upon success. """ self.queue_service = QueueService(account_name, account_key) self.queue_service.create_queue(self.queue_name) def close(self): """Perform any necessary clearnup on the queue at the end of a run. """ if self.queue_type == "AzureStorageQueue": pass elif self.queue_type == "LocalFile": self.file_queue.close() else: self.log.error("Unknown queue type: " + queue_type) def enqueue(self, msg, level="INFO"): msg = level + " - " + msg if self.queue_type == "LocalFile": file_queue.write(msg + "\n") elif self.queue_type == "AzureStorageQueue": self.queue_service.put_message(self.queue_name, msg) self.log.debug(msg) def dequeue(self): messages = [] if self.queue_type == "LocalFile": with open(config.UNPROCESSED_LOG_FILE, "r") as f: messages = f.readlines()[1] elif self.queue_type == "AzureStorageQueue": messages = self.queue_service.get_messages(self.queue_name) return messages def delete(self, message): self.queue_service.delete_message(self.queue_name, message.message_id, message.pop_receipt) # with open(config.PROCESSED_LOG_FILE, 'a') as processed: # processed.write(log) # os.remove(config.UNPROCESSED_LOG_FILE) def delete_queue(self, queue_name): self.queue_service.delete_queue(queue_name, False) def getLength(self): """ Get the approximate length of the queue """ queue_metadata = self.queue_service.get_queue_metadata(self.queue_name) count = queue_metadata["x-ms-approximate-messages-count"] return count
} stor_acc_name = service_keys['stor_acc_name'] stor_acc_key = service_keys['stor_acc_key'] redis_pass = service_keys['redis_pass'] redis_server = service_keys['redis_server'] instr_key = service_keys['instr_key'] # storage account_name = stor_acc_name account_key = stor_acc_key blob_service = BlobService(account_name, account_key) blob_service.create_container('images') queue_service = QueueService(account_name, account_key) queue_service.create_queue('taskqueue') table_service = TableService(account_name, account_key) table_service.create_table('tasktable') r = redis.StrictRedis(host=redis_server, port=6380, db=0, password=redis_pass, ssl=True) tc = TelemetryClient(instr_key) @app.route('/') @app.route('/home') def form(): return render_template('form_submit.html') @app.route('/hello/', methods=['POST'])
class Queue: def __init__(self, account_name, account_key, queue_name="logqueue"): """Initialiaze a queue. The type is set by the 'ACS_LOGGING_QUEUE_TYPE' environment variable. If it is set to 'AzureStorageQueue' then values must be provided for 'account_name' and 'account_key' which are values associated with the Azure Storage account. 'queue_name' is optional and defaults to 'logqueue'. """ self.log = Log() self.queue_type = config.ACS_LOGGING_QUEUE_TYPE self.queue_name = queue_name # self.log.debug("Queue type: " + self.queue_type + " / " + self.queue_name) if self.queue_type == "AzureStorageQueue": self.createAzureQueues(account_name, account_key) elif self.queue_type == "LocalFile": self.file_queue = open(config.UNPROCESSED_LOG_FILE, 'w+') else: self.log.error("Unknown queue type: " + queue_type) def getName(self): return self.queue_name def createAzureQueues(self, account_name, account_key): """ Create a queue for unprocessed log messages. Entries in the queue will be dequeued, processed and deleted upon success. """ self.queue_service = QueueService(account_name, account_key) self.queue_service.create_queue(self.queue_name) def close(self): """Perform any necessary clearnup on the queue at the end of a run. """ if self.queue_type == "AzureStorageQueue": pass elif self.queue_type == "LocalFile": self.file_queue.close() else: self.log.error("Unknown queue type: " + queue_type) def enqueue(self, msg): if self.queue_type == "LocalFile": file_queue.write(msg + '\n') elif self.queue_type == "AzureStorageQueue": self.queue_service.put_message(self.queue_name, msg) else: self.log.error("We don't know how to handle queues of type " + self.queue_type) self.log.debug(msg) def dequeue(self): messages = [] if self.queue_type == "LocalFile": with open(config.UNPROCESSED_LOG_FILE, 'r') as f: messages = f.readlines()[1] elif self.queue_type == "AzureStorageQueue": messages = self.queue_service.get_messages(self.queue_name) return messages def delete(self, message): self.queue_service.delete_message(self.queue_name, message.message_id, message.pop_receipt) # with open(config.PROCESSED_LOG_FILE, 'a') as processed: # processed.write(log) # os.remove(config.UNPROCESSED_LOG_FILE) def delete_queue(self, queue_name): self.queue_service.delete_queue(queue_name, False) def getLength(self): """ Get the approximate length of the queue """ queue_metadata = self.queue_service.get_queue_metadata(self.queue_name) count = queue_metadata['x-ms-approximate-messages-count'] return int(count) def peek_messages(self, num_messages): """ Peek at the top messages in the queue. This method does not remove the messages from the queue. """ return self.queue_service.peek_messages(self.queue_name, num_messages)
class MainPawWorker: """ Main class to use for running a worker. call start_workers() to start. """ def __init__(self, azure_storage_name, azure_storage_private_key, azure_queue_name, azure_table_name, tasks_module, workers, visibility_timeout=VISIBILITY_TIMEOUT): """ :param azure_storage_name: Name of Azure storage account :param azure_storage_private_key: Private key of Azure storage account. :param azure_queue_name: Name of the Azure queue to use. :param azure_table_name: Name of the Azure table to use. :param tasks_module: Module containing decorated functions to load from. :param workers: Int of workers. Ex: 4 :param visibility_timeout: Seconds in int to keep message in Azure queue """ self.account_name = azure_storage_name self.account_key = azure_storage_private_key self.queue_name = azure_queue_name self.table_name = azure_table_name self.tasks_module = tasks_module self.workers = workers self.visibility_timeout = visibility_timeout if self.visibility_timeout > MAXIMUM_VISIBILITY_TIMEOUT: raise PawError('self.visibility_timeout bigger than allowed limit') self.queue_service = QueueService(account_name=self.account_name, account_key=self.account_key) self.table_service = TableService(account_name=self.account_name, account_key=self.account_key) self.local_queue = Queue(self.workers) # self.logger = logging.getLogger() self.logger = LOGGER self.logger.info(PAW_LOGO) self.worker_process = Worker( local_queue=self.local_queue, queue_service=self.queue_service, queue_name=self.queue_name, table_service=self.table_service, table_name=azure_table_name, tasks=self._load_tasks(), ) self.pool = Pool(self.workers, self.worker_process.run, ()) signal.signal(signal.SIGTERM, self.on_exit) def on_exit(self, signum, frame): self.pool.terminate() sys.exit() def _load_tasks(self): """ Loads and returns decorated functions from a given modules, as a dict """ tasks = dict([ o for o in getmembers(self.tasks_module) if isfunction(o[1]) and hasattr(o[1], 'paw') ]) for t, f in tasks.items(): self.logger.info("REGISTERED '{}'".format(t)) if f.description: self.logger.info("\tdescription: '{}'".format(f.description)) if not tasks: self.logger.warning("No tasks found...") return tasks def start_workers(self, sleep_for=5): """ Starts workers and picks message from the Azure queue. On new message, when the local queue has room, the message is placed for a worker to pick-up :param sleep_for: Seconds to sleep for after a loop end. """ self.queue_service.create_queue(self.queue_name) create_table_if_missing(self.table_service, self.table_name) try: self.logger.info( "Cleaning up dead jobs left in {}".format(STARTED)) dead_jobs = self.table_service.query_entities( table_name=self.table_name, filter="status eq '{}'".format(STARTED)) for job in dead_jobs.items: log_to_table(table_service=self.table_service, table_name=self.table_name, message=job, status=LOST_WORKER, result="Lost worker, or task aborted.") except AzureException as e: self.logger.error("Cleaning dead tasks failed: {}".format(e)) while True: if self.local_queue.full(): time.sleep(sleep_for) try: new_msg = self.queue_service.get_messages( queue_name=self.queue_name, num_messages=1, visibility_timeout=self.visibility_timeout) except AzureException: self.logger.error("Error while getting message " "from Azure queue. Trying to create " "the queue") self.queue_service.create_queue(self.queue_name) time.sleep(sleep_for) continue if new_msg: msg = new_msg[0] try: content = json.loads(msg.content) except json.JSONDecodeError: self.logger.critical('Json error {}'.format( traceback.format_exc())) try: self.queue_service.delete_message( queue_name=self.queue_name, message_id=msg.id, pop_receipt=msg.pop_receipt) except AzureException: self.logger.critical( 'Deleting invalid message from queue failed: ' '{}'.format(traceback.format_exc())) continue if msg.dequeue_count > MAXIMUM_DEQUEUE_COUNT: log_to_table( table_service=self.table_service, table_name=self.table_name, message=content, status=FAILED, result="PAW MESSAGE: Dequeue count exceeded.", ) self.queue_service.delete_message(self.queue_name, msg.id, msg.pop_receipt) continue content['msg'] = msg while True: try: self.local_queue.put_nowait(content) break except Full: self.logger.info('LOCAL QUEUE FULL: waiting...') time.sleep(sleep_for) self.logger.debug('ADDING: {}'.format(content['task_name'])) time.sleep(sleep_for)
def create_queue_from_storage_account(storage_account, name): keys = StorageUtilities.get_storage_keys(storage_account.id) queue_service = QueueService(account_name=storage_account.name, account_key=keys[0].value) return queue_service.create_queue(name)
# Main thread - Check if port 25565 and move to the next if it's not. Once a free port has been found, create a new instance (up to 3) while instanceCreated == 0: freeport = checkPort(defaultport) if freeport == 0: defaultport += 1 counter += 1 if counter > maxInstances: print "Too many instances! you can't run more than %s instances" % maxInstances sys.exit() else: if debug == 1: print "Port %s is free" % defaultport container = createMcInstance(defaultport) instanceCreated = 1 counter +=1 # Return Instance ID, port and total number of instances once finished creating the server print "Instance ID: %s" % container["Id"] print "Listening on port %s" % defaultport print "There are %s instances running on this server" % counter # Update servers queue queue_service = QueueService(account_name, account_key) try: queue_service.create_queue(queuename) except: print "Queue creation failed." queue_service.put_message(queuename, '{"ContainerID":"%s", "ContainerPort":"%s", "ServerID":"%s"}' % container["Id"], defaultport, socket.gethostname()) sys.exit()
def get_queue_service(): "Returns the QueueService that can be used to put, peek, get and delete messages" queue_service = QueueService(ACCOUNT_NAME, ACCOUNT_KEY) queue_service.create_queue(QUEUE_NAME) return queue_service
PUT_URL = BASEURL + "/api/zip/site/wwwroot" SOURCE_DIR = os.path.dirname(os.path.abspath(__file__)) file_like_object = io.BytesIO() zipfile_obj = zipfile.ZipFile(file_like_object, mode='w') for f in ['function.json', 'run.py', 'jobrunner.py']: f = os.path.abspath(os.path.join(SOURCE_DIR, f)) a = os.path.relpath(f, SOURCE_DIR + "/..") zipfile_obj.write(f, arcname=a) zipfile_obj.writestr("host.json", "{}") zipfile_obj.close() r = put(PUT_URL, auth = (KUDU_USER, KUDU_PASS), data=file_like_object.getvalue()) print r.text print "funciion deployed" queue_service = QueueService(account_name=AZURE_STORAGE_ACC_NAME, account_key=AZURE_STORAGE_KEY) queue_service.create_queue("pywrenqueue") print "queue deployed" blob_service = BlockBlobService(account_name=AZURE_STORAGE_ACC_NAME, account_key=AZURE_STORAGE_KEY) blob_service.create_container("pywren1") print "container created"
logger = logging.getLogger(loggerName) init_logger() try: azureQueueAccountName = "" azureQueueKey = "" azureQueueAnalysisRecognizedItems = "recognizeditems-processing" azureQueueAnalysisResults = "analysis-results" azureQueueImageResults = "image-processing" queue_service = QueueService(account_name=azureQueueAccountName, account_key=azureQueueKey) while True: #create queue if doesnt exist if not queue_service.exists(azureQueueAnalysisRecognizedItems): queue_service.create_queue(azureQueueAnalysisRecognizedItems) if not queue_service.exists(azureQueueImageResults): queue_service.create_queue(azureQueueImageResults) #get queue count metadata = queue_service.get_queue_metadata( azureQueueAnalysisResults) queue_count = metadata.approximate_message_count if queue_count > 0: read_next_in_queue() else: logger.info("time.sleep(3000)") time.sleep(3000)