def add_to_initial_crawling_queue(name_list):
mycol = refer_collection()
connect_str = os.getenv('AZURE_STORAGE_CONNECTION_STRING')
ic_client = QueueClient.from_connection_string(connect_str, "initial-crawling-queue")
for name in name_list:
print(name)
ic_client.send_message([str(name)])
def test_peek_messages(self, resource_group, location, storage_account, storage_account_key):
# Instantiate a queue client
from azure.storage.queue import QueueClient
queue = QueueClient.from_connection_string(self.connection_string(storage_account, storage_account_key), "peekqueue")
# Create the queue
queue.create_queue()
try:
# Send messages
queue.send_message(u"message1")
queue.send_message(u"message2")
queue.send_message(u"message3")
queue.send_message(u"message4")
queue.send_message(u"message5")
# [START peek_message]
# Peek at one message at the front of the queue
msg = queue.peek_messages()
# Peek at the last 5 messages
messages = queue.peek_messages(max_messages=5)
# Print the last 5 messages
for message in messages:
print(message.content)
# [END peek_message]
finally:
# Delete the queue
queue.delete_queue()
def update_message(self):
# Instantiate a queue client
from azure.storage.queue import QueueClient
queue = QueueClient.from_connection_string(self.connection_string,
"myqueue6")
# Create the queue
queue.create_queue()
try:
# [START update_message]
# Send a message
queue.send_message(u"update me")
# Receive the message
messages = queue.receive_messages()
# Update the message
list_result = next(messages)
message = queue.update_message(list_result.id,
pop_receipt=list_result.pop_receipt,
visibility_timeout=0,
content=u"updated")
# [END update_message]
finally:
# Delete the queue
queue.delete_queue()
def list_message_pages(self):
# Instantiate a queue client
from azure.storage.queue import QueueClient
queue = QueueClient.from_connection_string(self.connection_string,
"myqueue4")
# Create the queue
queue.create_queue()
try:
queue.send_message(u"message1")
queue.send_message(u"message2")
queue.send_message(u"message3")
queue.send_message(u"message4")
queue.send_message(u"message5")
queue.send_message(u"message6")
# [START receive_messages_listing]
# Store two messages in each page
message_batches = queue.receive_messages(
messages_per_page=2).by_page()
# Iterate through the page lists
print(list(next(message_batches)))
print(list(next(message_batches)))
# There are two iterations in the last page as well.
last_page = next(message_batches)
for message in last_page:
print(message)
# [END receive_messages_listing]
finally:
queue.delete_queue()
def test_delete_and_clear_messages(self, resource_group, location, storage_account, storage_account_key):
# Instantiate a queue client
from azure.storage.queue import QueueClient
queue = QueueClient.from_connection_string(self.connection_string(storage_account, storage_account_key), "delqueue")
# Create the queue
queue.create_queue()
try:
# Send messages
queue.send_message(u"message1")
queue.send_message(u"message2")
queue.send_message(u"message3")
queue.send_message(u"message4")
queue.send_message(u"message5")
# [START delete_message]
# Get the message at the front of the queue
msg = next(queue.receive_messages())
# Delete the specified message
queue.delete_message(msg)
# [END delete_message]
# [START clear_messages]
queue.clear_messages()
# [END clear_messages]
finally:
# Delete the queue
queue.delete_queue()
def test_queue_and_messages_example(self, resource_group, location, storage_account, storage_account_key):
# Instantiate the QueueClient from a connection string
from azure.storage.queue import QueueClient
queue = QueueClient.from_connection_string(self.connection_string(storage_account, storage_account_key), "myqueue")
# Create the queue
# [START create_queue]
queue.create_queue()
# [END create_queue]
try:
# Enqueue messages
queue.enqueue_message(u"I'm using queues!")
queue.enqueue_message(u"This is my second message")
# Receive the messages
response = queue.receive_messages(messages_per_page=2)
# Print the content of the messages
for message in response:
print(message.content)
finally:
# [START delete_queue]
queue.delete_queue()
def test_queue_metadata(self, resource_group, location, storage_account,
storage_account_key):
# Instantiate a queue client
from azure.storage.queue import QueueClient
queue = QueueClient.from_connection_string(
self.connection_string(storage_account, storage_account_key),
"metaqueue")
# Create the queue
queue.create_queue()
try:
# [START set_queue_metadata]
metadata = {'foo': 'val1', 'bar': 'val2', 'baz': 'val3'}
queue.set_queue_metadata(metadata=metadata)
# [END set_queue_metadata]
# [START get_queue_properties]
response = queue.get_queue_properties().metadata
# [END get_queue_properties]
assert response == metadata
finally:
# Delete the queue
queue.delete_queue()
def main(msg: func.QueueMessage) -> None:
queue_client = QueueClient.from_connection_string(
os.environ['AzureWebJobsStorage'],
'oat-queue',
message_encode_policy=TextBase64EncodePolicy(),
)
queue_client.send_message(msg.get_body().decode(), visibility_timeout=3600)
def receive_one_message_from_queue(self):
# Instantiate a queue client
from azure.storage.queue import QueueClient
queue = QueueClient.from_connection_string(self.connection_string,
"myqueue5")
# Create the queue
queue.create_queue()
try:
queue.send_message(u"message1")
queue.send_message(u"message2")
queue.send_message(u"message3")
# [START receive_one_message]
# Pop two messages from the front of the queue
message1 = queue.receive_message()
message2 = queue.receive_message()
# We should see message 3 if we peek
message3 = queue.peek_messages()[0]
print(message1.content)
print(message2.content)
print(message3.content)
# [END receive_one_message]
finally:
queue.delete_queue()
def test_update_message(self, resource_group, location, storage_account, storage_account_key):
# Instantiate a queue client
from azure.storage.queue import QueueClient
queue = QueueClient.from_connection_string(self.connection_string(storage_account, storage_account_key), "updatequeue")
# Create the queue
queue.create_queue()
try:
# [START update_message]
# Send a message
queue.send_message(u"update me")
# Receive the message
messages = queue.receive_messages()
# Update the message
list_result = next(messages)
message = queue.update_message(
list_result.id,
pop_receipt=list_result.pop_receipt,
visibility_timeout=0,
content=u"updated")
# [END update_message]
assert message.content == "updated"
finally:
# Delete the queue
queue.delete_queue()
def add_to_project_completion_queue(id_list):
connect_str = os.getenv('AZURE_STORAGE_CONNECTION_STRING')
projects_client = QueueClient.from_connection_string(connect_str, "project-completion-queue")
for each_id in id_list:
print(each_id," added to projects completion queue")
projects_client.send_message([str(each_id)], time_to_live=-1)
def queue_and_messages_example(self):
# Instantiate the QueueClient from a connection string
from azure.storage.queue import QueueClient
queue = QueueClient.from_connection_string(
conn_str=self.connection_string, queue_name="myqueue")
# Create the queue
# [START create_queue]
queue.create_queue()
# [END create_queue]
try:
# Send messages
queue.send_message(u"I'm using queues!")
queue.send_message(u"This is my second message")
# Receive the messages
response = queue.receive_messages(messages_per_page=2)
# Print the content of the messages
for message in response:
print(message.content)
finally:
# [START delete_queue]
queue.delete_queue()
def get_projects_via_queue():
print("Projects queue is live")
mycol = refer_projects_col()
connect_str = os.getenv('AZURE_STORAGE_CONNECTION_STRING')
projects_client = QueueClient.from_connection_string(connect_str, "projects-queue")
while (True):
time.sleep(10)
rows = projects_client.receive_messages()
# print(rows)
for msg in rows:
# time.sleep(60)
row = msg.content
row = ast.literal_eval(row)
print(row[0],' processing queries from the key phrases')
entry_id = ObjectId(row[0])
proj_data_entry = mycol.find({"_id": entry_id})
data = [i for i in proj_data_entry]
# print(data[0])
key_phrases = data[0]['key_phrases']
queries = process_queries(key_phrases)
query_count = len(queries)
mycol.update_one({'_id': entry_id },
{'$set': {'query_count': query_count}})
for each_query in queries:
print(each_query," adding to pipeline execution")
add_to_initial_crawling_queue([each_query+' ++'+str(entry_id)+' --project'])
projects_client.delete_message(msg)
add_to_project_completion_queue([entry_id])
def main(req: func.HttpRequest, cmdl: bytes, cvec: bytes, clbl: str,
nmdl: bytes, guid: str) -> func.HttpResponse:
logging.info('Executing GraphQL function.')
queue = QueueClient.from_connection_string(
os.environ['AzureWebJobsStorage'], 'predictions')
try:
query = req.get_body().decode()
except ValueError:
pass
if query:
schema = Schema(Query)
context = CACHE_MANAGER.get(guid, cmdl, cvec, clbl, nmdl)
results = schema.execute(query, context=context)
response = responses.graphql(results)
# Write response to azure queue storage
message = responses.storage(results)
if message:
queue.send_message(message, time_to_live=-1)
return response
else:
return responses.bad_request(
'Please pass a GraphQL query in the request body.')
def delete_and_clear_messages(self):
# Instantiate a queue client
from azure.storage.queue import QueueClient
queue = QueueClient.from_connection_string(self.connection_string,
"myqueue4")
# Create the queue
queue.create_queue()
try:
# Send messages
queue.send_message(u"message1")
queue.send_message(u"message2")
queue.send_message(u"message3")
queue.send_message(u"message4")
queue.send_message(u"message5")
# [START delete_message]
# Get the message at the front of the queue
msg = next(queue.receive_messages())
# Delete the specified message
queue.delete_message(msg)
# [END delete_message]
# [START clear_messages]
queue.clear_messages()
# [END clear_messages]
finally:
# Delete the queue
queue.delete_queue()
def peek_messages(self):
# Instantiate a queue client
from azure.storage.queue import QueueClient
queue = QueueClient.from_connection_string(self.connection_string,
"myqueue5")
# Create the queue
queue.create_queue()
try:
# Send messages
queue.send_message(u"message1")
queue.send_message(u"message2")
queue.send_message(u"message3")
queue.send_message(u"message4")
queue.send_message(u"message5")
# [START peek_message]
# Peek at one message at the front of the queue
msg = queue.peek_messages()
# Peek at the last 5 messages
messages = queue.peek_messages(max_messages=5)
# Print the last 5 messages
for message in messages:
print(message.content)
# [END peek_message]
finally:
# Delete the queue
queue.delete_queue()
def setup_azure(azure_connection_string, queue_name):
try:
queue_client = QueueClient.from_connection_string(conn_str=azure_connection_string, queue_name=queue_name)
blob_service_client = BlobServiceClient.from_connection_string(azure_connection_string)
except Exception as e:
logger.exception(e)
return (queue_client, blob_service_client)
def __init__(self, name: str):
self.conn_str = settings.QUEUE_CONN_STRING
self.name = f"{settings.ENVIRONMENT}-{name}"
self.client = QueueClient.from_connection_string(self.conn_str, self.name)
try:
self.client.create_queue()
except ResourceExistsError:
pass
def add_to_projects_queue(id_list):
connect_str = os.getenv('AZURE_STORAGE_CONNECTION_STRING')
connect_str = "DefaultEndpointsProtocol=https;AccountName=armitage;AccountKey=yUoQAb2ZRKKFiQBzMUTLKd1YSNbd0zjkgFaAz9OS9ze+RJW6DWbeeDsPmNfucyXlDEEGmU6WUlv36My2RARLLA==;EndpointSuffix=core.windows.net"
projects_client = QueueClient.from_connection_string(connect_str, "projects-queue")
for each_id in id_list:
print(each_id," added to projects queue")
projects_client.send_message([str(each_id)], time_to_live=-1)
def add_to_query_queue(id_list):
connect_str = os.getenv('AZURE_STORAGE_CONNECTION_STRING')
query_client = QueueClient.from_connection_string(connect_str,
"query-queue")
for each_id in id_list:
print(each_id, " added to query queue")
query_client.send_message([str(each_id)], time_to_live=-1)
def test_set_access_policy(self, resource_group, location, storage_account, storage_account_key):
# SAS URL is calculated from storage key, so this test runs live only
if not self.is_live:
return
# [START create_queue_client_from_connection_string]
from azure.storage.queue import QueueClient
queue_client = QueueClient.from_connection_string(self.connection_string(storage_account, storage_account_key), "queuetest")
# [END create_queue_client_from_connection_string]
# Create the queue
try:
queue_client.create_queue()
except ResourceExistsError:
pass
queue_client.send_message(u"hello world")
try:
# [START set_access_policy]
# Create an access policy
from azure.storage.queue import AccessPolicy, QueueSasPermissions
access_policy = AccessPolicy()
access_policy.start = datetime.utcnow() - timedelta(hours=1)
access_policy.expiry = datetime.utcnow() + timedelta(hours=1)
access_policy.permission = QueueSasPermissions(read=True)
identifiers = {'my-access-policy-id': access_policy}
# Set the access policy
queue_client.set_queue_access_policy(identifiers)
# [END set_access_policy]
# Use the access policy to generate a SAS token
# [START queue_client_sas_token]
from azure.storage.queue import generate_queue_sas
sas_token = generate_queue_sas(
queue_client.account_name,
queue_client.queue_name,
queue_client.credential.account_key,
policy_id='my-access-policy-id'
)
# [END queue_client_sas_token]
# Authenticate with the sas token
# [START create_queue_client]
q = QueueClient.from_queue_url(
queue_url=queue_client.url,
credential=sas_token
)
# [END create_queue_client]
# Use the newly authenticated client to receive messages
my_message = q.receive_messages()
assert my_message is not None
finally:
# Delete the queue
queue_client.delete_queue()
def queue_client():
queue_name = "apm-agent-python-ci-" + str(uuid.uuid4())
queue_client = QueueClient.from_connection_string(CONNECTION_STRING,
queue_name)
queue_client.create_queue()
yield queue_client
queue_client.delete_queue()
def send_chunk_info(self, export_job_id, chunk_id):
with QueueClient.from_connection_string(
self.connection_string,
self.queue_name,
message_encode_policy=BinaryBase64EncodePolicy(),
message_decode_policy=BinaryBase64DecodePolicy(
)) as queue_client:
chunk_info = {'exportJobId': export_job_id, 'chunkId': chunk_id}
return queue_client.send_message(
json.dumps(chunk_info).encode('utf-8'))
def __init__(self, csv_path, output_facility_csv):
self._csv_file = csv_path
self._csv_columns = []
self._facility_output = facility_csv
_queue = 'waterbeacon'
_connection_string = '' #TODO load from config
self._queue_client = QueueClient.from_connection_string(
conn_str=_connection_string, queue_name=_queue)
def main(myblob: func.InputStream):
logging.info(f"Python blob trigger function processed blob \n"
f"Name: {myblob.name}\n"
f"Blob Size: {myblob.length} bytes")
connection_string = os.environ["snibirkedastor_STORAGE"]
blob_name = myblob.name[10:]
queue_client = QueueClient.from_connection_string(connection_string, "canny-queue")
encoded_bytes = base64.b64encode(blob_name.encode("utf-8"))
encoded_str = str(encoded_bytes, "utf-8")
queue_client.send_message(encoded_str)
def create(self):
with QueueClient.from_connection_string(
self.connection_string,
self.queue_name,
message_encode_policy=BinaryBase64EncodePolicy(),
message_decode_policy=BinaryBase64DecodePolicy(
)) as queue_client:
try:
queue_client.create_queue()
except ResourceExistsError:
logging.warn(f'Queue {self.queue_name} already exists')
def test_set_access_policy(self):
# SAS URL is calculated from storage key, so this test runs live only
if TestMode.need_recording_file(self.test_mode):
return
# [START create_queue_client_from_connection_string]
from azure.storage.queue import QueueClient
queue_client = QueueClient.from_connection_string(self.connection_string, "queuetest")
# [END create_queue_client_from_connection_string]
# Create the queue
try:
queue_client.create_queue()
except ResourceExistsError:
pass
queue_client.enqueue_message(u"hello world")
try:
# [START set_access_policy]
# Create an access policy
from azure.storage.queue import AccessPolicy, QueuePermissions
access_policy = AccessPolicy()
access_policy.start = datetime.utcnow() - timedelta(hours=1)
access_policy.expiry = datetime.utcnow() + timedelta(hours=1)
access_policy.permission = QueuePermissions.READ
identifiers = {'my-access-policy-id': access_policy}
# Set the access policy
queue_client.set_queue_access_policy(identifiers)
# [END set_access_policy]
# Use the access policy to generate a SAS token
# [START queue_client_sas_token]
sas_token = queue_client.generate_shared_access_signature(
policy_id='my-access-policy-id'
)
# [END queue_client_sas_token]
# Authenticate with the sas token
# [START create_queue_client]
q = QueueClient(
queue_url=queue_client.url,
credential=sas_token
)
# [END create_queue_client]
# Use the newly authenticated client to receive messages
my_message = q.receive_messages()
assert my_message is not None
finally:
# Delete the queue
queue_client.delete_queue()
def query_state_update_via_queue():
print("Query state updating queue is live")
query_collection = refer_query_col()
mycol = refer_collection()
connect_str = os.getenv('AZURE_STORAGE_CONNECTION_STRING')
query_client = QueueClient.from_connection_string(connect_str,
"query-queue")
while (True):
# print('q')
time.sleep(200)
rows = query_client.receive_messages()
for msg in rows:
time.sleep(10)
row = msg.content
print(row)
row = ast.literal_eval(row)
print('getting_id', row[0])
entry_id = ObjectId(row[0])
query_data_entry = query_collection.find({"_id": entry_id})
data = [i for i in query_data_entry]
#check_for_the_completion_of_components
try:
associated_entries = data[0]['associated_entries']
print('getting associated entries')
completed_count = 0
for each_entry_res in associated_entries:
res_entry = mycol.find({"_id": each_entry_res})
# print("profile",each_entry_res)
data_res = [i for i in res_entry]
if (data_res[0]['simplified_dump_state'] == 'Completed'):
completed_count += 1
print('completed_count', completed_count)
print('entry_count', data[0]['entry_count'])
if (completed_count == data[0]['entry_count']):
print("All the entries are completed for the query",
completed_count)
query_collection.update_one(
{'_id': entry_id}, {'$set': {
'state': 'Completed'
}})
query_client.delete_message(msg)
except KeyError as e:
print('Query is not yet ready', e)
except IndexError as e:
print('Yet query entry not available')
except Exception as e:
print("Exception Occured during dumping ", e)
def set_access_policy(self):
# [START create_queue_client_from_connection_string]
from azure.storage.queue import QueueClient
queue = QueueClient.from_connection_string(self.connection_string, "myqueue1")
# [END create_queue_client_from_connection_string]
# Create the queue
queue.create_queue()
# Send a message
queue.send_message(u"hello world")
try:
# [START set_access_policy]
# Create an access policy
from azure.storage.queue import AccessPolicy, QueueSasPermissions
access_policy = AccessPolicy()
access_policy.start = datetime.utcnow() - timedelta(hours=1)
access_policy.expiry = datetime.utcnow() + timedelta(hours=1)
access_policy.permission = QueueSasPermissions(read=True)
identifiers = {'my-access-policy-id': access_policy}
# Set the access policy
queue.set_queue_access_policy(identifiers)
# [END set_access_policy]
# Use the access policy to generate a SAS token
# [START queue_client_sas_token]
from azure.storage.queue import generate_queue_sas
sas_token = generate_queue_sas(
queue.account_name,
queue.queue_name,
queue.credential.account_key,
policy_id='my-access-policy-id'
)
# [END queue_client_sas_token]
# Authenticate with the sas token
# [START create_queue_client]
token_auth_queue = QueueClient.from_queue_url(
queue_url=queue.url,
credential=sas_token
)
# [END create_queue_client]
# Use the newly authenticated client to receive messages
my_message = token_auth_queue.receive_messages()
finally:
# Delete the queue
queue.delete_queue()
def create_queue_client(account_name, account_key, queue_name):
queue_client = None
try:
connection_string = f"DefaultEndpointsProtocol=https;AccountName={account_name};AccountKey={account_key};EndpointSuffix=core.windows.net"
queue_client = QueueClient.from_connection_string(
connection_string, queue_name)
logging.info(
f"Created queue client for queue {queue_name} in account {account_name}."
)
except Exception as e:
logging.error(
f"Could not create queue client for queue {queue_name} in account {account_name}: {e}"
)
return queue_client