def add_to_initial_crawling_queue(name_list): mycol = refer_collection() connect_str = os.getenv('AZURE_STORAGE_CONNECTION_STRING') ic_client = QueueClient.from_connection_string(connect_str, "initial-crawling-queue") for name in name_list: print(name) ic_client.send_message([str(name)])
def test_peek_messages(self, resource_group, location, storage_account, storage_account_key): # Instantiate a queue client from azure.storage.queue import QueueClient queue = QueueClient.from_connection_string(self.connection_string(storage_account, storage_account_key), "peekqueue") # Create the queue queue.create_queue() try: # Send messages queue.send_message(u"message1") queue.send_message(u"message2") queue.send_message(u"message3") queue.send_message(u"message4") queue.send_message(u"message5") # [START peek_message] # Peek at one message at the front of the queue msg = queue.peek_messages() # Peek at the last 5 messages messages = queue.peek_messages(max_messages=5) # Print the last 5 messages for message in messages: print(message.content) # [END peek_message] finally: # Delete the queue queue.delete_queue()
def update_message(self): # Instantiate a queue client from azure.storage.queue import QueueClient queue = QueueClient.from_connection_string(self.connection_string, "myqueue6") # Create the queue queue.create_queue() try: # [START update_message] # Send a message queue.send_message(u"update me") # Receive the message messages = queue.receive_messages() # Update the message list_result = next(messages) message = queue.update_message(list_result.id, pop_receipt=list_result.pop_receipt, visibility_timeout=0, content=u"updated") # [END update_message] finally: # Delete the queue queue.delete_queue()
def list_message_pages(self): # Instantiate a queue client from azure.storage.queue import QueueClient queue = QueueClient.from_connection_string(self.connection_string, "myqueue4") # Create the queue queue.create_queue() try: queue.send_message(u"message1") queue.send_message(u"message2") queue.send_message(u"message3") queue.send_message(u"message4") queue.send_message(u"message5") queue.send_message(u"message6") # [START receive_messages_listing] # Store two messages in each page message_batches = queue.receive_messages( messages_per_page=2).by_page() # Iterate through the page lists print(list(next(message_batches))) print(list(next(message_batches))) # There are two iterations in the last page as well. last_page = next(message_batches) for message in last_page: print(message) # [END receive_messages_listing] finally: queue.delete_queue()
def test_delete_and_clear_messages(self, resource_group, location, storage_account, storage_account_key): # Instantiate a queue client from azure.storage.queue import QueueClient queue = QueueClient.from_connection_string(self.connection_string(storage_account, storage_account_key), "delqueue") # Create the queue queue.create_queue() try: # Send messages queue.send_message(u"message1") queue.send_message(u"message2") queue.send_message(u"message3") queue.send_message(u"message4") queue.send_message(u"message5") # [START delete_message] # Get the message at the front of the queue msg = next(queue.receive_messages()) # Delete the specified message queue.delete_message(msg) # [END delete_message] # [START clear_messages] queue.clear_messages() # [END clear_messages] finally: # Delete the queue queue.delete_queue()
def test_queue_and_messages_example(self, resource_group, location, storage_account, storage_account_key): # Instantiate the QueueClient from a connection string from azure.storage.queue import QueueClient queue = QueueClient.from_connection_string(self.connection_string(storage_account, storage_account_key), "myqueue") # Create the queue # [START create_queue] queue.create_queue() # [END create_queue] try: # Enqueue messages queue.enqueue_message(u"I'm using queues!") queue.enqueue_message(u"This is my second message") # Receive the messages response = queue.receive_messages(messages_per_page=2) # Print the content of the messages for message in response: print(message.content) finally: # [START delete_queue] queue.delete_queue()
def test_queue_metadata(self, resource_group, location, storage_account, storage_account_key): # Instantiate a queue client from azure.storage.queue import QueueClient queue = QueueClient.from_connection_string( self.connection_string(storage_account, storage_account_key), "metaqueue") # Create the queue queue.create_queue() try: # [START set_queue_metadata] metadata = {'foo': 'val1', 'bar': 'val2', 'baz': 'val3'} queue.set_queue_metadata(metadata=metadata) # [END set_queue_metadata] # [START get_queue_properties] response = queue.get_queue_properties().metadata # [END get_queue_properties] assert response == metadata finally: # Delete the queue queue.delete_queue()
def main(msg: func.QueueMessage) -> None: queue_client = QueueClient.from_connection_string( os.environ['AzureWebJobsStorage'], 'oat-queue', message_encode_policy=TextBase64EncodePolicy(), ) queue_client.send_message(msg.get_body().decode(), visibility_timeout=3600)
def receive_one_message_from_queue(self): # Instantiate a queue client from azure.storage.queue import QueueClient queue = QueueClient.from_connection_string(self.connection_string, "myqueue5") # Create the queue queue.create_queue() try: queue.send_message(u"message1") queue.send_message(u"message2") queue.send_message(u"message3") # [START receive_one_message] # Pop two messages from the front of the queue message1 = queue.receive_message() message2 = queue.receive_message() # We should see message 3 if we peek message3 = queue.peek_messages()[0] print(message1.content) print(message2.content) print(message3.content) # [END receive_one_message] finally: queue.delete_queue()
def test_update_message(self, resource_group, location, storage_account, storage_account_key): # Instantiate a queue client from azure.storage.queue import QueueClient queue = QueueClient.from_connection_string(self.connection_string(storage_account, storage_account_key), "updatequeue") # Create the queue queue.create_queue() try: # [START update_message] # Send a message queue.send_message(u"update me") # Receive the message messages = queue.receive_messages() # Update the message list_result = next(messages) message = queue.update_message( list_result.id, pop_receipt=list_result.pop_receipt, visibility_timeout=0, content=u"updated") # [END update_message] assert message.content == "updated" finally: # Delete the queue queue.delete_queue()
def add_to_project_completion_queue(id_list): connect_str = os.getenv('AZURE_STORAGE_CONNECTION_STRING') projects_client = QueueClient.from_connection_string(connect_str, "project-completion-queue") for each_id in id_list: print(each_id," added to projects completion queue") projects_client.send_message([str(each_id)], time_to_live=-1)
def queue_and_messages_example(self): # Instantiate the QueueClient from a connection string from azure.storage.queue import QueueClient queue = QueueClient.from_connection_string( conn_str=self.connection_string, queue_name="myqueue") # Create the queue # [START create_queue] queue.create_queue() # [END create_queue] try: # Send messages queue.send_message(u"I'm using queues!") queue.send_message(u"This is my second message") # Receive the messages response = queue.receive_messages(messages_per_page=2) # Print the content of the messages for message in response: print(message.content) finally: # [START delete_queue] queue.delete_queue()
def get_projects_via_queue(): print("Projects queue is live") mycol = refer_projects_col() connect_str = os.getenv('AZURE_STORAGE_CONNECTION_STRING') projects_client = QueueClient.from_connection_string(connect_str, "projects-queue") while (True): time.sleep(10) rows = projects_client.receive_messages() # print(rows) for msg in rows: # time.sleep(60) row = msg.content row = ast.literal_eval(row) print(row[0],' processing queries from the key phrases') entry_id = ObjectId(row[0]) proj_data_entry = mycol.find({"_id": entry_id}) data = [i for i in proj_data_entry] # print(data[0]) key_phrases = data[0]['key_phrases'] queries = process_queries(key_phrases) query_count = len(queries) mycol.update_one({'_id': entry_id }, {'$set': {'query_count': query_count}}) for each_query in queries: print(each_query," adding to pipeline execution") add_to_initial_crawling_queue([each_query+' ++'+str(entry_id)+' --project']) projects_client.delete_message(msg) add_to_project_completion_queue([entry_id])
def main(req: func.HttpRequest, cmdl: bytes, cvec: bytes, clbl: str, nmdl: bytes, guid: str) -> func.HttpResponse: logging.info('Executing GraphQL function.') queue = QueueClient.from_connection_string( os.environ['AzureWebJobsStorage'], 'predictions') try: query = req.get_body().decode() except ValueError: pass if query: schema = Schema(Query) context = CACHE_MANAGER.get(guid, cmdl, cvec, clbl, nmdl) results = schema.execute(query, context=context) response = responses.graphql(results) # Write response to azure queue storage message = responses.storage(results) if message: queue.send_message(message, time_to_live=-1) return response else: return responses.bad_request( 'Please pass a GraphQL query in the request body.')
def delete_and_clear_messages(self): # Instantiate a queue client from azure.storage.queue import QueueClient queue = QueueClient.from_connection_string(self.connection_string, "myqueue4") # Create the queue queue.create_queue() try: # Send messages queue.send_message(u"message1") queue.send_message(u"message2") queue.send_message(u"message3") queue.send_message(u"message4") queue.send_message(u"message5") # [START delete_message] # Get the message at the front of the queue msg = next(queue.receive_messages()) # Delete the specified message queue.delete_message(msg) # [END delete_message] # [START clear_messages] queue.clear_messages() # [END clear_messages] finally: # Delete the queue queue.delete_queue()
def peek_messages(self): # Instantiate a queue client from azure.storage.queue import QueueClient queue = QueueClient.from_connection_string(self.connection_string, "myqueue5") # Create the queue queue.create_queue() try: # Send messages queue.send_message(u"message1") queue.send_message(u"message2") queue.send_message(u"message3") queue.send_message(u"message4") queue.send_message(u"message5") # [START peek_message] # Peek at one message at the front of the queue msg = queue.peek_messages() # Peek at the last 5 messages messages = queue.peek_messages(max_messages=5) # Print the last 5 messages for message in messages: print(message.content) # [END peek_message] finally: # Delete the queue queue.delete_queue()
def setup_azure(azure_connection_string, queue_name): try: queue_client = QueueClient.from_connection_string(conn_str=azure_connection_string, queue_name=queue_name) blob_service_client = BlobServiceClient.from_connection_string(azure_connection_string) except Exception as e: logger.exception(e) return (queue_client, blob_service_client)
def __init__(self, name: str): self.conn_str = settings.QUEUE_CONN_STRING self.name = f"{settings.ENVIRONMENT}-{name}" self.client = QueueClient.from_connection_string(self.conn_str, self.name) try: self.client.create_queue() except ResourceExistsError: pass
def add_to_projects_queue(id_list): connect_str = os.getenv('AZURE_STORAGE_CONNECTION_STRING') connect_str = "DefaultEndpointsProtocol=https;AccountName=armitage;AccountKey=yUoQAb2ZRKKFiQBzMUTLKd1YSNbd0zjkgFaAz9OS9ze+RJW6DWbeeDsPmNfucyXlDEEGmU6WUlv36My2RARLLA==;EndpointSuffix=core.windows.net" projects_client = QueueClient.from_connection_string(connect_str, "projects-queue") for each_id in id_list: print(each_id," added to projects queue") projects_client.send_message([str(each_id)], time_to_live=-1)
def add_to_query_queue(id_list): connect_str = os.getenv('AZURE_STORAGE_CONNECTION_STRING') query_client = QueueClient.from_connection_string(connect_str, "query-queue") for each_id in id_list: print(each_id, " added to query queue") query_client.send_message([str(each_id)], time_to_live=-1)
def test_set_access_policy(self, resource_group, location, storage_account, storage_account_key): # SAS URL is calculated from storage key, so this test runs live only if not self.is_live: return # [START create_queue_client_from_connection_string] from azure.storage.queue import QueueClient queue_client = QueueClient.from_connection_string(self.connection_string(storage_account, storage_account_key), "queuetest") # [END create_queue_client_from_connection_string] # Create the queue try: queue_client.create_queue() except ResourceExistsError: pass queue_client.send_message(u"hello world") try: # [START set_access_policy] # Create an access policy from azure.storage.queue import AccessPolicy, QueueSasPermissions access_policy = AccessPolicy() access_policy.start = datetime.utcnow() - timedelta(hours=1) access_policy.expiry = datetime.utcnow() + timedelta(hours=1) access_policy.permission = QueueSasPermissions(read=True) identifiers = {'my-access-policy-id': access_policy} # Set the access policy queue_client.set_queue_access_policy(identifiers) # [END set_access_policy] # Use the access policy to generate a SAS token # [START queue_client_sas_token] from azure.storage.queue import generate_queue_sas sas_token = generate_queue_sas( queue_client.account_name, queue_client.queue_name, queue_client.credential.account_key, policy_id='my-access-policy-id' ) # [END queue_client_sas_token] # Authenticate with the sas token # [START create_queue_client] q = QueueClient.from_queue_url( queue_url=queue_client.url, credential=sas_token ) # [END create_queue_client] # Use the newly authenticated client to receive messages my_message = q.receive_messages() assert my_message is not None finally: # Delete the queue queue_client.delete_queue()
def queue_client(): queue_name = "apm-agent-python-ci-" + str(uuid.uuid4()) queue_client = QueueClient.from_connection_string(CONNECTION_STRING, queue_name) queue_client.create_queue() yield queue_client queue_client.delete_queue()
def send_chunk_info(self, export_job_id, chunk_id): with QueueClient.from_connection_string( self.connection_string, self.queue_name, message_encode_policy=BinaryBase64EncodePolicy(), message_decode_policy=BinaryBase64DecodePolicy( )) as queue_client: chunk_info = {'exportJobId': export_job_id, 'chunkId': chunk_id} return queue_client.send_message( json.dumps(chunk_info).encode('utf-8'))
def __init__(self, csv_path, output_facility_csv): self._csv_file = csv_path self._csv_columns = [] self._facility_output = facility_csv _queue = 'waterbeacon' _connection_string = '' #TODO load from config self._queue_client = QueueClient.from_connection_string( conn_str=_connection_string, queue_name=_queue)
def main(myblob: func.InputStream): logging.info(f"Python blob trigger function processed blob \n" f"Name: {myblob.name}\n" f"Blob Size: {myblob.length} bytes") connection_string = os.environ["snibirkedastor_STORAGE"] blob_name = myblob.name[10:] queue_client = QueueClient.from_connection_string(connection_string, "canny-queue") encoded_bytes = base64.b64encode(blob_name.encode("utf-8")) encoded_str = str(encoded_bytes, "utf-8") queue_client.send_message(encoded_str)
def create(self): with QueueClient.from_connection_string( self.connection_string, self.queue_name, message_encode_policy=BinaryBase64EncodePolicy(), message_decode_policy=BinaryBase64DecodePolicy( )) as queue_client: try: queue_client.create_queue() except ResourceExistsError: logging.warn(f'Queue {self.queue_name} already exists')
def test_set_access_policy(self): # SAS URL is calculated from storage key, so this test runs live only if TestMode.need_recording_file(self.test_mode): return # [START create_queue_client_from_connection_string] from azure.storage.queue import QueueClient queue_client = QueueClient.from_connection_string(self.connection_string, "queuetest") # [END create_queue_client_from_connection_string] # Create the queue try: queue_client.create_queue() except ResourceExistsError: pass queue_client.enqueue_message(u"hello world") try: # [START set_access_policy] # Create an access policy from azure.storage.queue import AccessPolicy, QueuePermissions access_policy = AccessPolicy() access_policy.start = datetime.utcnow() - timedelta(hours=1) access_policy.expiry = datetime.utcnow() + timedelta(hours=1) access_policy.permission = QueuePermissions.READ identifiers = {'my-access-policy-id': access_policy} # Set the access policy queue_client.set_queue_access_policy(identifiers) # [END set_access_policy] # Use the access policy to generate a SAS token # [START queue_client_sas_token] sas_token = queue_client.generate_shared_access_signature( policy_id='my-access-policy-id' ) # [END queue_client_sas_token] # Authenticate with the sas token # [START create_queue_client] q = QueueClient( queue_url=queue_client.url, credential=sas_token ) # [END create_queue_client] # Use the newly authenticated client to receive messages my_message = q.receive_messages() assert my_message is not None finally: # Delete the queue queue_client.delete_queue()
def query_state_update_via_queue(): print("Query state updating queue is live") query_collection = refer_query_col() mycol = refer_collection() connect_str = os.getenv('AZURE_STORAGE_CONNECTION_STRING') query_client = QueueClient.from_connection_string(connect_str, "query-queue") while (True): # print('q') time.sleep(200) rows = query_client.receive_messages() for msg in rows: time.sleep(10) row = msg.content print(row) row = ast.literal_eval(row) print('getting_id', row[0]) entry_id = ObjectId(row[0]) query_data_entry = query_collection.find({"_id": entry_id}) data = [i for i in query_data_entry] #check_for_the_completion_of_components try: associated_entries = data[0]['associated_entries'] print('getting associated entries') completed_count = 0 for each_entry_res in associated_entries: res_entry = mycol.find({"_id": each_entry_res}) # print("profile",each_entry_res) data_res = [i for i in res_entry] if (data_res[0]['simplified_dump_state'] == 'Completed'): completed_count += 1 print('completed_count', completed_count) print('entry_count', data[0]['entry_count']) if (completed_count == data[0]['entry_count']): print("All the entries are completed for the query", completed_count) query_collection.update_one( {'_id': entry_id}, {'$set': { 'state': 'Completed' }}) query_client.delete_message(msg) except KeyError as e: print('Query is not yet ready', e) except IndexError as e: print('Yet query entry not available') except Exception as e: print("Exception Occured during dumping ", e)
def set_access_policy(self): # [START create_queue_client_from_connection_string] from azure.storage.queue import QueueClient queue = QueueClient.from_connection_string(self.connection_string, "myqueue1") # [END create_queue_client_from_connection_string] # Create the queue queue.create_queue() # Send a message queue.send_message(u"hello world") try: # [START set_access_policy] # Create an access policy from azure.storage.queue import AccessPolicy, QueueSasPermissions access_policy = AccessPolicy() access_policy.start = datetime.utcnow() - timedelta(hours=1) access_policy.expiry = datetime.utcnow() + timedelta(hours=1) access_policy.permission = QueueSasPermissions(read=True) identifiers = {'my-access-policy-id': access_policy} # Set the access policy queue.set_queue_access_policy(identifiers) # [END set_access_policy] # Use the access policy to generate a SAS token # [START queue_client_sas_token] from azure.storage.queue import generate_queue_sas sas_token = generate_queue_sas( queue.account_name, queue.queue_name, queue.credential.account_key, policy_id='my-access-policy-id' ) # [END queue_client_sas_token] # Authenticate with the sas token # [START create_queue_client] token_auth_queue = QueueClient.from_queue_url( queue_url=queue.url, credential=sas_token ) # [END create_queue_client] # Use the newly authenticated client to receive messages my_message = token_auth_queue.receive_messages() finally: # Delete the queue queue.delete_queue()
def create_queue_client(account_name, account_key, queue_name): queue_client = None try: connection_string = f"DefaultEndpointsProtocol=https;AccountName={account_name};AccountKey={account_key};EndpointSuffix=core.windows.net" queue_client = QueueClient.from_connection_string( connection_string, queue_name) logging.info( f"Created queue client for queue {queue_name} in account {account_name}." ) except Exception as e: logging.error( f"Could not create queue client for queue {queue_name} in account {account_name}: {e}" ) return queue_client