def test_run_policy_for_event(self, add_job_mock, _0, _1, _2, _3):
host = Host(DEFAULT_EVENT_QUEUE_ID, DEFAULT_EVENT_QUEUE_NAME,
DEFAULT_POLICY_STORAGE)
host.policies = {
'one': {
'policy':
ContainerHostTest.get_mock_policy({
'name': 'one',
'mode': {
'type': 'container-event',
'events': ['ResourceGroupWrite', 'VnetWrite']
}
})
}
}
message = QueueMessage()
message.id = 1
message.dequeue_count = 0
message.content = \
"""eyAgCiAgICJzdWJqZWN0IjoiL3N1YnNjcmlwdGlvbnMvZWE5ODk3NGItNWQyYS00ZDk4LWE3OGEt
MzgyZjM3MTVkMDdlL3Jlc291cmNlR3JvdXBzL3Rlc3RfY29udGFpbmVyX21vZGUiLAogICAiZXZl
bnRUeXBlIjoiTWljcm9zb2Z0LlJlc291cmNlcy5SZXNvdXJjZVdyaXRlU3VjY2VzcyIsCiAgICJl
dmVudFRpbWUiOiIyMDE5LTA3LTE2VDE4OjMwOjQzLjM1OTUyNTVaIiwKICAgImlkIjoiNjE5ZDI2
NzQtYjM5Ni00MzU2LTk2MTktNmM1YTUyZmU0ZTg4IiwKICAgImRhdGEiOnsgICAgICAgIAogICAg
ICAiY29ycmVsYXRpb25JZCI6IjdkZDVhNDc2LWUwNTItNDBlMi05OWU0LWJiOTg1MmRjMWY4NiIs
CiAgICAgICJyZXNvdXJjZVByb3ZpZGVyIjoiTWljcm9zb2Z0LlJlc291cmNlcyIsCiAgICAgICJy
ZXNvdXJjZVVyaSI6Ii9zdWJzY3JpcHRpb25zL2VhOTg5NzRiLTVkMmEtNGQ5OC1hNzhhLTM4MmYz
NzE1ZDA3ZS9yZXNvdXJjZUdyb3Vwcy90ZXN0X2NvbnRhaW5lcl9tb2RlIiwKICAgICAgIm9wZXJh
dGlvbk5hbWUiOiJNaWNyb3NvZnQuUmVzb3VyY2VzL3N1YnNjcmlwdGlvbnMvcmVzb3VyY2VHcm91
cHMvd3JpdGUiLAogICAgICAic3RhdHVzIjoiU3VjY2VlZGVkIgogICB9LAogICAidG9waWMiOiIv
c3Vic2NyaXB0aW9ucy9hYTk4OTc0Yi01ZDJhLTRkOTgtYTc4YS0zODJmMzcxNWQwN2UiCn0="""
# run with real match
host.run_policies_for_event(message)
add_job_mock.assert_called_with(
ANY,
id='one619d2674-b396-4356-9619-6c5a52fe4e88',
name='one',
args=ANY,
misfire_grace_time=ANY)
add_job_mock.reset_mock()
# run with no match
host.policies = {}
host.run_policies_for_event(message)
self.assertFalse(add_job_mock.called)
async def peek_messages(self,
max_messages=None,
timeout=None,
**kwargs): # type: ignore
# type: (Optional[int], Optional[int], Optional[Any]) -> List[QueueMessage]
"""Retrieves one or more messages from the front of the queue, but does
not alter the visibility of the message.
Only messages that are visible may be retrieved. When a message is retrieved
for the first time with a call to :func:`~receive_messages`, its dequeue_count property
is set to 1. If it is not deleted and is subsequently retrieved again, the
dequeue_count property is incremented. The client may use this value to
determine how many times a message has been retrieved. Note that a call
to peek_messages does not increment the value of DequeueCount, but returns
this value for the client to read.
If the key-encryption-key or resolver field is set on the local service object,
the messages will be decrypted before being returned.
:param int max_messages:
A nonzero integer value that specifies the number of
messages to peek from the queue, up to a maximum of 32. By default,
a single message is peeked from the queue with this operation.
:param int timeout:
The server timeout, expressed in seconds.
:return:
A list of :class:`~azure.storage.queue.QueueMessage` objects. Note that
time_next_visible and pop_receipt will not be populated as peek does
not pop the message and can only retrieve already visible messages.
:rtype: list(:class:`~azure.storage.queue.QueueMessage`)
.. admonition:: Example:
.. literalinclude:: ../tests/test_queue_samples_message_async.py
:start-after: [START async_peek_message]
:end-before: [END async_peek_message]
:language: python
:dedent: 12
:caption: Peek messages.
"""
if max_messages and not 1 <= max_messages <= 32:
raise ValueError(
"Number of messages to peek should be between 1 and 32")
self._config.message_decode_policy.configure(
require_encryption=self.require_encryption,
key_encryption_key=self.key_encryption_key,
resolver=self.key_resolver_function)
try:
messages = await self._client.messages.peek(
number_of_messages=max_messages,
timeout=timeout,
cls=self._config.message_decode_policy,
**kwargs)
wrapped_messages = []
for peeked in messages:
wrapped_messages.append(QueueMessage._from_generated(peeked)) # pylint: disable=protected-access
return wrapped_messages
except StorageErrorException as error:
process_storage_error(error)
def mock_message(success):
m = QueueMessage()
m.id = uuid4()
m.insertion_time = time.time()
m.expiration_time = None
m.dequeue_count = None
if success:
content = {
"OperationId": str(m.id),
"Database": "db1",
"Table": "table1",
"IngestionSourceId": str(m.id),
"IngestionSourcePath": "blob/path",
"RootActivityId": "1",
"SucceededOn": time.time(),
}
else:
content = {
"OperationId": str(m.id),
"Database": "db1",
"Table": "table1",
"IngestionSourceId": str(m.id),
"IngestionSourcePath": "blob/path",
"RootActivityId": "1",
"FailedOn": time.time(),
"Details": "",
"ErrorCode": "1",
"FailureStatus": "",
"OriginatesFromUpdatePolicy": "",
"ShouldRetry": False,
}
m.content = str(
base64.b64encode(
json.dumps(content).encode("utf-8")).decode("utf-8"))
m.pop_receipt = None
m.time_next_visible = None
return m
def test_isempty(self, mocked_q_peek_messages):
client = KustoIngestClient("some-cluster")
with mock.patch.object(
client._resource_manager, "get_successful_ingestions_queues"
) as mocked_get_success_qs, mock.patch.object(
client._resource_manager,
"get_failed_ingestions_queues") as mocked_get_failed_qs:
fake_failed_queue = _ResourceUri("mocked_storage_account1",
"queue", "mocked_qf_name",
"mocked_sas")
fake_success_queue = _ResourceUri("mocked_storage_account2",
"queue", "mocked_qs_name",
"mocked_sas")
mocked_get_success_qs.return_value = [fake_success_queue]
mocked_get_failed_qs.return_value = [fake_failed_queue]
mocked_q_peek_messages.side_effect = (
lambda queue_name, num_messages=1: []
if queue_name == fake_failed_queue.object_name else
[QueueMessage() for _ in range(0, num_messages)])
qs = KustoIngestStatusQueues(client)
assert qs.success.is_empty() == False
assert qs.failure.is_empty() == True
assert mocked_q_peek_messages.call_count == 2
assert mocked_q_peek_messages.call_args_list[0][0][
0] == fake_success_queue.object_name
assert mocked_q_peek_messages.call_args_list[0][1][
"num_messages"] == 2
assert mocked_q_peek_messages.call_args_list[1][0][
0] == fake_failed_queue.object_name
assert mocked_q_peek_messages.call_args_list[1][1][
"num_messages"] == 2
def test_poll_queue(self, run_policy_mock, storage_mock, _1, _2, _3):
host = Host(DEFAULT_EVENT_QUEUE_ID, DEFAULT_EVENT_QUEUE_NAME,
DEFAULT_POLICY_STORAGE)
host.policies = {
'one': {
'policy':
ContainerHostTest.get_mock_policy({
'name': 'one',
'mode': {
'type': 'container-event',
'events': ['ResourceGroupWrite', 'VnetWrite']
}
})
}
}
q1 = QueueMessage()
q1.id = 1
q1.dequeue_count = 0
q1.content = """eyAgCiAgICJzdWJqZWN0IjoiL3N1YnNjcmlwdGlvbnMvZWE5ODk3NGItNWQyYS00ZDk4LWE3
OGEtMzgyZjM3MTVkMDdlL3Jlc291cmNlR3JvdXBzL3Rlc3RfY29udGFpbmVyX21vZGUiLAogICAiZXZlbnRUeXBlIj
oiTWljcm9zb2Z0LlJlc291cmNlcy5SZXNvdXJjZVdyaXRlU3VjY2VzcyIsCiAgICJldmVudFRpbWUiOiIyMDE5LTA3
LTE2VDE4OjMwOjQzLjM1OTUyNTVaIiwKICAgImlkIjoiNjE5ZDI2NzQtYjM5Ni00MzU2LTk2MTktNmM1YTUyZmU0ZT
g4IiwKICAgImRhdGEiOnsgICAgICAgIAogICAgICAiY29ycmVsYXRpb25JZCI6IjdkZDVhNDc2LWUwNTItNDBlMi05
OWU0LWJiOTg1MmRjMWY4NiIsCiAgICAgICJyZXNvdXJjZVByb3ZpZGVyIjoiTWljcm9zb2Z0LlJlc291cmNlcyIsCi
AgICAgICJyZXNvdXJjZVVyaSI6Ii9zdWJzY3JpcHRpb25zL2VhOTg5NzRiLTVkMmEtNGQ5OC1hNzhhLTM4MmYzNzE1
ZDA3ZS9yZXNvdXJjZUdyb3Vwcy90ZXN0X2NvbnRhaW5lcl9tb2RlIiwKICAgICAgIm9wZXJhdGlvbk5hbWUiOiJNaW
Nyb3NvZnQuUmVzb3VyY2VzL3N1YnNjcmlwdGlvbnMvcmVzb3VyY2VHcm91cHMvd3JpdGUiLAogICAgICAic3RhdHVz
IjoiU3VjY2VlZGVkIiwKICAgfSwKICAgInRvcGljIjoiL3N1YnNjcmlwdGlvbnMvYWE5ODk3NGItNWQyYS00ZDk4LW
E3OGEtMzgyZjM3MTVkMDdlIgp9"""
q2 = QueueMessage()
q2.id = 2
q2.dequeue_count = 0
q2.content = q1.content
# Return 2 messages on first call, then none
storage_mock.get_queue_messages.side_effect = [[q1, q2], []]
host.poll_queue()
self.assertEqual(2, run_policy_mock.call_count)
run_policy_mock.reset_mock()
# Return 5 messages on first call, then 2, then 0
storage_mock.get_queue_messages.side_effect = [[q1, q1, q1, q1, q1],
[q1, q2], []]
host.poll_queue()
self.assertEqual(7, run_policy_mock.call_count)
run_policy_mock.reset_mock()
# High dequeue count
q1.dequeue_count = 100
storage_mock.get_queue_messages.side_effect = [[q1, q2], []]
host.poll_queue()
self.assertEqual(1, run_policy_mock.call_count)
async def update_message(
self,
message,
visibility_timeout=None,
pop_receipt=None, # type: ignore
content=None,
timeout=None,
**kwargs):
# type: (Any, int, Optional[str], Optional[Any], Optional[int], Any) -> QueueMessage
"""Updates the visibility timeout of a message. You can also use this
operation to update the contents of a message.
This operation can be used to continually extend the invisibility of a
queue message. This functionality can be useful if you want a worker role
to "lease" a queue message. For example, if a worker role calls :func:`~receive_messages()`
and recognizes that it needs more time to process a message, it can
continually extend the message's invisibility until it is processed. If
the worker role were to fail during processing, eventually the message
would become visible again and another worker role could process it.
If the key-encryption-key field is set on the local service object, this method will
encrypt the content before uploading.
:param str message:
The message object or id identifying the message to update.
:param int visibility_timeout:
Specifies the new visibility timeout value, in seconds,
relative to server time. The new value must be larger than or equal
to 0, and cannot be larger than 7 days. The visibility timeout of a
message cannot be set to a value later than the expiry time. A
message can be updated until it has been deleted or has expired.
The message object or message id identifying the message to update.
:param str pop_receipt:
A valid pop receipt value returned from an earlier call
to the :func:`~receive_messages` or :func:`~update_message` operation.
:param obj content:
Message content. Allowed type is determined by the encode_function
set on the service. Default is str.
:param int timeout:
The server timeout, expressed in seconds.
:return:
A :class:`~azure.storage.queue.models.QueueMessage` object. For convenience,
this object is also populated with the content, although it is not returned by the service.
:rtype: ~azure.storage.queue.models.QueueMessage
Example:
.. literalinclude:: ../tests/test_queue_samples_message.py
:start-after: [START update_message]
:end-before: [END update_message]
:language: python
:dedent: 12
:caption: Update a message.
"""
try:
message_id = message.id
message_text = content or message.content
receipt = pop_receipt or message.pop_receipt
insertion_time = message.insertion_time
expiration_time = message.expiration_time
dequeue_count = message.dequeue_count
except AttributeError:
message_id = message
message_text = content
receipt = pop_receipt
insertion_time = None
expiration_time = None
dequeue_count = None
if receipt is None:
raise ValueError("pop_receipt must be present")
if message_text is not None:
self._config.message_encode_policy.configure(
self.require_encryption, self.key_encryption_key,
self.key_resolver_function)
message_text = self._config.message_encode_policy(message_text)
updated = GenQueueMessage(message_text=message_text)
else:
updated = None # type: ignore
try:
response = await self._client.message_id.update(
queue_message=updated,
visibilitytimeout=visibility_timeout or 0,
timeout=timeout,
pop_receipt=receipt,
cls=return_response_headers,
queue_message_id=message_id,
**kwargs)
new_message = QueueMessage(content=message_text)
new_message.id = message_id
new_message.insertion_time = insertion_time
new_message.expiration_time = expiration_time
new_message.dequeue_count = dequeue_count
new_message.pop_receipt = response["popreceipt"]
new_message.time_next_visible = response["time_next_visible"]
return new_message
except StorageErrorException as error:
process_storage_error(error)
async def enqueue_message( # type: ignore
self,
content, # type: Any
visibility_timeout=None, # type: Optional[int]
time_to_live=None, # type: Optional[int]
timeout=None, # type: Optional[int]
**kwargs # type: Optional[Any]
):
# type: (...) -> QueueMessage
"""Adds a new message to the back of the message queue.
The visibility timeout specifies the time that the message will be
invisible. After the timeout expires, the message will become visible.
If a visibility timeout is not specified, the default value of 0 is used.
The message time-to-live specifies how long a message will remain in the
queue. The message will be deleted from the queue when the time-to-live
period expires.
If the key-encryption-key field is set on the local service object, this method will
encrypt the content before uploading.
:param obj content:
Message content. Allowed type is determined by the encode_function
set on the service. Default is str. The encoded message can be up to
64KB in size.
:param int visibility_timeout:
If not specified, the default value is 0. Specifies the
new visibility timeout value, in seconds, relative to server time.
The value must be larger than or equal to 0, and cannot be
larger than 7 days. The visibility timeout of a message cannot be
set to a value later than the expiry time. visibility_timeout
should be set to a value smaller than the time-to-live value.
:param int time_to_live:
Specifies the time-to-live interval for the message, in
seconds. The time-to-live may be any positive number or -1 for infinity. If this
parameter is omitted, the default time-to-live is 7 days.
:param int timeout:
The server timeout, expressed in seconds.
:return:
A :class:`~azure.storage.queue.models.QueueMessage` object.
This object is also populated with the content although it is not
returned from the service.
:rtype: ~azure.storage.queue.models.QueueMessage
Example:
.. literalinclude:: ../tests/test_queue_samples_message.py
:start-after: [START enqueue_messages]
:end-before: [END enqueue_messages]
:language: python
:dedent: 12
:caption: Enqueue messages.
"""
self._config.message_encode_policy.configure(
self.require_encryption, self.key_encryption_key,
self.key_resolver_function)
content = self._config.message_encode_policy(content)
new_message = GenQueueMessage(message_text=content)
try:
enqueued = await self._client.messages.enqueue(
queue_message=new_message,
visibilitytimeout=visibility_timeout,
message_time_to_live=time_to_live,
timeout=timeout,
**kwargs)
queue_message = QueueMessage(content=new_message.message_text)
queue_message.id = enqueued[0].message_id
queue_message.insertion_time = enqueued[0].insertion_time
queue_message.expiration_time = enqueued[0].expiration_time
queue_message.pop_receipt = enqueued[0].pop_receipt
queue_message.time_next_visible = enqueued[0].time_next_visible
return queue_message
except StorageErrorException as error:
process_storage_error(error)
