async def _test_update_encrypted_binary_message(self):
# Arrange
queue = await self._create_queue()
queue.key_encryption_key = KeyWrapper('key1')
queue._config.message_encode_policy = BinaryBase64EncodePolicy()
queue._config.message_decode_policy = BinaryBase64DecodePolicy()
binary_message = self.get_random_bytes(100)
await queue.enqueue_message(binary_message)
messages = []
async for m in queue.receive_messages():
messages.append(m)
list_result1 = messages[0]
# Act
binary_message = self.get_random_bytes(100)
list_result1.content = binary_message
await queue.update_message(list_result1)
async for m in queue.receive_messages():
messages.append(m)
list_result2 = messages[0]
# Assert
self.assertEqual(binary_message, list_result2.content)
def test_update_encrypted_binary_message(self, resource_group, location,
storage_account,
storage_account_key):
# Arrange
qsc = QueueServiceClient(self.account_url(storage_account, "queue"),
storage_account_key)
queue = self._create_queue(
qsc,
message_encode_policy=BinaryBase64EncodePolicy(),
message_decode_policy=BinaryBase64DecodePolicy())
queue.key_encryption_key = KeyWrapper('key1')
binary_message = self.get_random_bytes(100)
queue.send_message(binary_message)
messages = []
for m in queue.receive_messages():
messages.append(m)
list_result1 = messages[0]
# Act
binary_message = self.get_random_bytes(100)
list_result1.content = binary_message
queue.update_message(list_result1)
for m in queue.receive_messages():
messages.append(m)
list_result2 = messages[0]
# Assert
self.assertEqual(binary_message, list_result2.content)
async def test_update_encrypted_binary_message(self, storage_account_name,
storage_account_key):
qsc = QueueServiceClient(self.account_url(storage_account_name,
"queue"),
storage_account_key,
transport=AiohttpTestTransport())
# Arrange
queue = await self._create_queue(
qsc,
message_encode_policy=BinaryBase64EncodePolicy(),
message_decode_policy=BinaryBase64DecodePolicy())
queue.key_encryption_key = KeyWrapper('key1')
binary_message = self.get_random_bytes(100)
await queue.send_message(binary_message)
messages = []
async for m in queue.receive_messages():
messages.append(m)
list_result1 = messages[0]
# Act
binary_message = self.get_random_bytes(100)
list_result1.content = binary_message
await queue.update_message(list_result1)
async for m in queue.receive_messages():
messages.append(m)
list_result2 = messages[0]
# Assert
self.assertEqual(binary_message, list_result2.content)
def send_chunk_info(self, export_job_id, chunk_id):
with QueueClient.from_connection_string(
self.connection_string,
self.queue_name,
message_encode_policy=BinaryBase64EncodePolicy(),
message_decode_policy=BinaryBase64DecodePolicy(
)) as queue_client:
chunk_info = {'exportJobId': export_job_id, 'chunkId': chunk_id}
return queue_client.send_message(
json.dumps(chunk_info).encode('utf-8'))
def create(self):
with QueueClient.from_connection_string(
self.connection_string,
self.queue_name,
message_encode_policy=BinaryBase64EncodePolicy(),
message_decode_policy=BinaryBase64DecodePolicy(
)) as queue_client:
try:
queue_client.create_queue()
except ResourceExistsError:
logging.warn(f'Queue {self.queue_name} already exists')
def test_message_bytes_base64(self, storage_account_name, storage_account_key):
# Arrange.
qsc = QueueServiceClient(self.account_url(storage_account_name, "queue"), storage_account_key)
queue = QueueClient(
account_url=self.account_url(storage_account_name, "queue"),
queue_name=self.get_resource_name(TEST_QUEUE_PREFIX),
credential=storage_account_key,
message_encode_policy=BinaryBase64EncodePolicy(),
message_decode_policy=BinaryBase64DecodePolicy())
message = b'xyz'
# Asserts
self._validate_encoding(queue, message)
def __init__(self, queue_name, queue_connection_string=None):
"""Initialize a JobQueue object."""
logger.debug("Initializing a JobQueue for [%s]", queue_name)
if queue_connection_string is None:
queue_connection_string = DEFAULT_QUEUE_CONNECTION_STRING
self._client = QueueClient.from_connection_string(
queue_connection_string, queue_name)
self._client.message_encode_policy = BinaryBase64EncodePolicy()
self._client.message_decode_policy = BinaryBase64DecodePolicy()
try:
self._client.create_queue()
except ResourceExistsError:
pass # OK, just means the queue already exists.
def test_message_bytes_base64(self):
# Arrange.
queue_url = self._get_queue_url()
credentials = self._get_shared_key_credential()
queue = QueueClient(
queue_url=queue_url,
queue=self.get_resource_name(TEST_QUEUE_PREFIX),
credential=credentials,
message_encode_policy=BinaryBase64EncodePolicy(),
message_decode_policy=BinaryBase64DecodePolicy())
message = b'xyz'
# Asserts
self._validate_encoding(queue, message)
def get_messages(connection_string, queue_name):
while True:
queue_client = QueueClient.from_connection_string(
connection_string, queue_name)
queue_client.message_encode_policy = BinaryBase64EncodePolicy()
queue_client.message_decode_policy = BinaryBase64DecodePolicy()
messages = queue_client.receive_messages()
for message in messages:
print("\nMessage Received: {}".format(
base64.b64decode(message.content).decode('utf-8')))
queue_client.delete_message(message.id, message.pop_receipt)
properties = queue_client.get_queue_properties()
if properties.approximate_message_count == 0:
time.sleep(1)
def test_message_text_fails(self, storage_account_name, storage_account_key):
# Arrange
qsc = QueueServiceClient(self.account_url(storage_account_name, "queue"), storage_account_key)
queue = QueueClient(
account_url=self.account_url(storage_account_name, "queue"),
queue_name=self.get_resource_name(TEST_QUEUE_PREFIX),
credential=storage_account_key,
message_encode_policy=BinaryBase64EncodePolicy(),
message_decode_policy=BinaryBase64DecodePolicy())
# Action.
with self.assertRaises(TypeError) as e:
message = u'xyz'
queue.send_message(message)
# Asserts
self.assertTrue(str(e.exception).startswith('Message content must be bytes'))
async def test_message_bytes_base64(self, resource_group, location,
storage_account, storage_account_key):
# Arrange.
qsc = QueueServiceClient(self._account_url(storage_account.name),
storage_account_key,
transport=AiohttpTestTransport())
queue = QueueClient(queue_url=self._account_url(storage_account.name),
queue=self.get_resource_name(TEST_QUEUE_PREFIX),
credential=storage_account_key,
message_encode_policy=BinaryBase64EncodePolicy(),
message_decode_policy=BinaryBase64DecodePolicy(),
transport=AiohttpTestTransport())
message = b'xyz'
# Asserts
await self._validate_encoding(queue, message)
def test_message_text_fails(self):
# Arrange
queue_url = self._get_queue_url()
credentials = self._get_shared_key_credential()
queue = QueueClient(
queue_url=queue_url,
queue=self.get_resource_name(TEST_QUEUE_PREFIX),
credential=credentials,
message_encode_policy=BinaryBase64EncodePolicy(),
message_decode_policy=BinaryBase64DecodePolicy())
# Action.
with self.assertRaises(TypeError) as e:
message = u'xyz'
queue.enqueue_message(message)
# Asserts
self.assertTrue(str(e.exception).startswith('Message content must be bytes'))
def initialize_queue(connection_string: str, queue_name: str) -> QueueClient:
"""Initialize a queue given the connection string and queue name."""
if connection_string is None or queue_name is None:
return None
client = None
try:
client = QueueClient.from_connection_string(connection_string, queue_name)
client.message_encode_policy = BinaryBase64EncodePolicy()
client.message_decode_policy = BinaryBase64DecodePolicy()
try:
client.create_queue()
except:
pass # OK to ignore
except:
client = None
return client
async def test_message_text_fails(self, resource_group, location,
storage_account, storage_account_key):
# Arrange
qsc = QueueServiceClient(self._account_url(storage_account.name),
storage_account_key,
transport=AiohttpTestTransport())
queue = QueueClient(queue_url=self._account_url(storage_account.name),
queue=self.get_resource_name(TEST_QUEUE_PREFIX),
credential=storage_account_key,
message_encode_policy=BinaryBase64EncodePolicy(),
message_decode_policy=BinaryBase64DecodePolicy(),
transport=AiohttpTestTransport())
# Action.
with self.assertRaises(TypeError) as e:
message = u'xyz'
await queue.enqueue_message(message)
# Asserts
self.assertTrue(
str(e.exception).startswith('Message content must be bytes'))
def test_update_encrypted_binary_message(self):
# Arrange
queue = self._create_queue()
queue.key_encryption_key = KeyWrapper('key1')
queue._config.message_encode_policy = BinaryBase64EncodePolicy()
queue._config.message_decode_policy = BinaryBase64DecodePolicy()
binary_message = self.get_random_bytes(100)
queue.enqueue_message(binary_message)
messages = queue.receive_messages()
list_result1 = next(messages)
# Act
binary_message = self.get_random_bytes(100)
list_result1.content = binary_message
queue.update_message(list_result1)
list_result2 = next(messages)
# Assert
self.assertEqual(binary_message, list_result2.content)
def step3():
with open('../local.settings.json') as fd:
settings = json.load(fd)
connectionString = settings["Values"]["AzureWebJobsStorage"]
#os.environ["AZURE_STORAGE_CONNECTION_STRING"] = connectionString
container_name = "opengameart"
with open('putToSqlQueue.txt') as fd:
files = [fname.strip() for fname in fd.readlines()]
#files = [fname for fname in files if fname.lower().endswith('.jpg') or fname.lower().endswith('.png')]
files = files[282000:]
# Retrieve the connection string from an environment
# variable named AZURE_STORAGE_CONNECTION_STRING
connect_str = os.getenv("AZURE_STORAGE_CONNECTION_STRING")
# Create a unique name for the queue
#q_name = "queue-" + str(uuid.uuid4())
q_name = 'sqlqueue'
# Instantiate a QueueClient object which will
# be used to create and manipulate the queue
#queue_client = QueueClient.from_connection_string(connectionString, q_name)
# Setup Base64 encoding and decoding functions
base64_queue_client = QueueClient.from_connection_string(
conn_str=connectionString, queue_name=q_name,
message_encode_policy = BinaryBase64EncodePolicy(),
message_decode_policy = BinaryBase64DecodePolicy()
)
for i, message in enumerate(files):
if i % 1000 == 0:
print(i)
base64_queue_client.send_message(message.encode('ascii'))
azure_storage_account = None
mongo_connect = None
queue = "test"
queue = "general-image-2-crawl"
cookies = []
with open("local.settings.json") as fin:
settings = json.load(fin)
azure_storage_account = settings.get("AzureStorageAccount")
mongo_connect = settings.get("MongoDBConnectionString")
if not azure_storage_account or not mongo_connect:
raise Exception("Null Settings on AzureStorageAccount or mongo connect")
# Setup Base64 encoding and decoding functions
base64_queue_client = QueueClient.from_connection_string(
conn_str=azure_storage_account, queue_name=queue,
message_encode_policy = BinaryBase64EncodePolicy(),
message_decode_policy = BinaryBase64DecodePolicy()
)
mongo_client = pymongo.MongoClient(mongo_connect)
mongo_db = 'dev'
mongo_collection = "mingju5"
mongo_docs = mongo_client[mongo_db][mongo_collection]
with open("data/mingju.csv", 'r', encoding='utf-8') as fin:
fin.readline()
for idx, line in enumerate(fin):
if idx < 4747:
continue
gs = line.split(",")
assert len(gs) == 4
# Info for authenticationg
# Display name : huyeduon-restapi
# Application (client) ID : c4f505e2-b31b-42ad-8294-11991044cece
# Directory (tenant) ID : e34b6079-b8db-43a4-8104-4768856bd06c
# Object ID : 52a4d658-867e-41fa-947d-2f51befd0f2d
tenant_id = "e34b6079-b8db-43a4-8104-4768856bd06c"
client_id = "c4f505e2-b31b-42ad-8294-11991044cece"
client_secret = "E9LpKrW-rqFabdc3-2aMSML4Hq~-k.R.rw"
# Get the application credentials
app_credentials = ClientSecretCredential(tenant_id, client_id, client_secret)
# Create a queue client
queue_client = QueueClient.from_connection_string(connect_str,
processing_queue_name,
credential=app_credentials)
# Setup Base64 encoding and decoding functions
queue_client.message_encode_policy = BinaryBase64EncodePolicy()
queue_client.message_decode_policy = BinaryBase64DecodePolicy()
print("Client connected to Queue")
print("Peek messages in the queue...")
messages = queue_client.peek_messages(max_messages=5)
for peeked_message in messages:
print("Message: ", base64ToString(peeked_message.content))
