def test_run_policy_for_event(self, add_job_mock, _0, _1, _2, _3): host = Host(DEFAULT_EVENT_QUEUE_ID, DEFAULT_EVENT_QUEUE_NAME, DEFAULT_POLICY_STORAGE) host.policies = { 'one': { 'policy': ContainerHostTest.get_mock_policy({ 'name': 'one', 'mode': { 'type': 'container-event', 'events': ['ResourceGroupWrite', 'VnetWrite'] } }) } } message = QueueMessage() message.id = 1 message.dequeue_count = 0 message.content = \ """eyAgCiAgICJzdWJqZWN0IjoiL3N1YnNjcmlwdGlvbnMvZWE5ODk3NGItNWQyYS00ZDk4LWE3OGEt MzgyZjM3MTVkMDdlL3Jlc291cmNlR3JvdXBzL3Rlc3RfY29udGFpbmVyX21vZGUiLAogICAiZXZl bnRUeXBlIjoiTWljcm9zb2Z0LlJlc291cmNlcy5SZXNvdXJjZVdyaXRlU3VjY2VzcyIsCiAgICJl dmVudFRpbWUiOiIyMDE5LTA3LTE2VDE4OjMwOjQzLjM1OTUyNTVaIiwKICAgImlkIjoiNjE5ZDI2 NzQtYjM5Ni00MzU2LTk2MTktNmM1YTUyZmU0ZTg4IiwKICAgImRhdGEiOnsgICAgICAgIAogICAg ICAiY29ycmVsYXRpb25JZCI6IjdkZDVhNDc2LWUwNTItNDBlMi05OWU0LWJiOTg1MmRjMWY4NiIs CiAgICAgICJyZXNvdXJjZVByb3ZpZGVyIjoiTWljcm9zb2Z0LlJlc291cmNlcyIsCiAgICAgICJy ZXNvdXJjZVVyaSI6Ii9zdWJzY3JpcHRpb25zL2VhOTg5NzRiLTVkMmEtNGQ5OC1hNzhhLTM4MmYz NzE1ZDA3ZS9yZXNvdXJjZUdyb3Vwcy90ZXN0X2NvbnRhaW5lcl9tb2RlIiwKICAgICAgIm9wZXJh dGlvbk5hbWUiOiJNaWNyb3NvZnQuUmVzb3VyY2VzL3N1YnNjcmlwdGlvbnMvcmVzb3VyY2VHcm91 cHMvd3JpdGUiLAogICAgICAic3RhdHVzIjoiU3VjY2VlZGVkIgogICB9LAogICAidG9waWMiOiIv c3Vic2NyaXB0aW9ucy9hYTk4OTc0Yi01ZDJhLTRkOTgtYTc4YS0zODJmMzcxNWQwN2UiCn0=""" # run with real match host.run_policies_for_event(message) add_job_mock.assert_called_with( ANY, id='one619d2674-b396-4356-9619-6c5a52fe4e88', name='one', args=ANY, misfire_grace_time=ANY) add_job_mock.reset_mock() # run with no match host.policies = {} host.run_policies_for_event(message) self.assertFalse(add_job_mock.called)
async def peek_messages(self, max_messages=None, timeout=None, **kwargs): # type: ignore # type: (Optional[int], Optional[int], Optional[Any]) -> List[QueueMessage] """Retrieves one or more messages from the front of the queue, but does not alter the visibility of the message. Only messages that are visible may be retrieved. When a message is retrieved for the first time with a call to :func:`~receive_messages`, its dequeue_count property is set to 1. If it is not deleted and is subsequently retrieved again, the dequeue_count property is incremented. The client may use this value to determine how many times a message has been retrieved. Note that a call to peek_messages does not increment the value of DequeueCount, but returns this value for the client to read. If the key-encryption-key or resolver field is set on the local service object, the messages will be decrypted before being returned. :param int max_messages: A nonzero integer value that specifies the number of messages to peek from the queue, up to a maximum of 32. By default, a single message is peeked from the queue with this operation. :param int timeout: The server timeout, expressed in seconds. :return: A list of :class:`~azure.storage.queue.QueueMessage` objects. Note that time_next_visible and pop_receipt will not be populated as peek does not pop the message and can only retrieve already visible messages. :rtype: list(:class:`~azure.storage.queue.QueueMessage`) .. admonition:: Example: .. literalinclude:: ../tests/test_queue_samples_message_async.py :start-after: [START async_peek_message] :end-before: [END async_peek_message] :language: python :dedent: 12 :caption: Peek messages. """ if max_messages and not 1 <= max_messages <= 32: raise ValueError( "Number of messages to peek should be between 1 and 32") self._config.message_decode_policy.configure( require_encryption=self.require_encryption, key_encryption_key=self.key_encryption_key, resolver=self.key_resolver_function) try: messages = await self._client.messages.peek( number_of_messages=max_messages, timeout=timeout, cls=self._config.message_decode_policy, **kwargs) wrapped_messages = [] for peeked in messages: wrapped_messages.append(QueueMessage._from_generated(peeked)) # pylint: disable=protected-access return wrapped_messages except StorageErrorException as error: process_storage_error(error)
def mock_message(success): m = QueueMessage() m.id = uuid4() m.insertion_time = time.time() m.expiration_time = None m.dequeue_count = None if success: content = { "OperationId": str(m.id), "Database": "db1", "Table": "table1", "IngestionSourceId": str(m.id), "IngestionSourcePath": "blob/path", "RootActivityId": "1", "SucceededOn": time.time(), } else: content = { "OperationId": str(m.id), "Database": "db1", "Table": "table1", "IngestionSourceId": str(m.id), "IngestionSourcePath": "blob/path", "RootActivityId": "1", "FailedOn": time.time(), "Details": "", "ErrorCode": "1", "FailureStatus": "", "OriginatesFromUpdatePolicy": "", "ShouldRetry": False, } m.content = str( base64.b64encode( json.dumps(content).encode("utf-8")).decode("utf-8")) m.pop_receipt = None m.time_next_visible = None return m
def test_isempty(self, mocked_q_peek_messages): client = KustoIngestClient("some-cluster") with mock.patch.object( client._resource_manager, "get_successful_ingestions_queues" ) as mocked_get_success_qs, mock.patch.object( client._resource_manager, "get_failed_ingestions_queues") as mocked_get_failed_qs: fake_failed_queue = _ResourceUri("mocked_storage_account1", "queue", "mocked_qf_name", "mocked_sas") fake_success_queue = _ResourceUri("mocked_storage_account2", "queue", "mocked_qs_name", "mocked_sas") mocked_get_success_qs.return_value = [fake_success_queue] mocked_get_failed_qs.return_value = [fake_failed_queue] mocked_q_peek_messages.side_effect = ( lambda queue_name, num_messages=1: [] if queue_name == fake_failed_queue.object_name else [QueueMessage() for _ in range(0, num_messages)]) qs = KustoIngestStatusQueues(client) assert qs.success.is_empty() == False assert qs.failure.is_empty() == True assert mocked_q_peek_messages.call_count == 2 assert mocked_q_peek_messages.call_args_list[0][0][ 0] == fake_success_queue.object_name assert mocked_q_peek_messages.call_args_list[0][1][ "num_messages"] == 2 assert mocked_q_peek_messages.call_args_list[1][0][ 0] == fake_failed_queue.object_name assert mocked_q_peek_messages.call_args_list[1][1][ "num_messages"] == 2
def test_poll_queue(self, run_policy_mock, storage_mock, _1, _2, _3): host = Host(DEFAULT_EVENT_QUEUE_ID, DEFAULT_EVENT_QUEUE_NAME, DEFAULT_POLICY_STORAGE) host.policies = { 'one': { 'policy': ContainerHostTest.get_mock_policy({ 'name': 'one', 'mode': { 'type': 'container-event', 'events': ['ResourceGroupWrite', 'VnetWrite'] } }) } } q1 = QueueMessage() q1.id = 1 q1.dequeue_count = 0 q1.content = """eyAgCiAgICJzdWJqZWN0IjoiL3N1YnNjcmlwdGlvbnMvZWE5ODk3NGItNWQyYS00ZDk4LWE3 OGEtMzgyZjM3MTVkMDdlL3Jlc291cmNlR3JvdXBzL3Rlc3RfY29udGFpbmVyX21vZGUiLAogICAiZXZlbnRUeXBlIj oiTWljcm9zb2Z0LlJlc291cmNlcy5SZXNvdXJjZVdyaXRlU3VjY2VzcyIsCiAgICJldmVudFRpbWUiOiIyMDE5LTA3 LTE2VDE4OjMwOjQzLjM1OTUyNTVaIiwKICAgImlkIjoiNjE5ZDI2NzQtYjM5Ni00MzU2LTk2MTktNmM1YTUyZmU0ZT g4IiwKICAgImRhdGEiOnsgICAgICAgIAogICAgICAiY29ycmVsYXRpb25JZCI6IjdkZDVhNDc2LWUwNTItNDBlMi05 OWU0LWJiOTg1MmRjMWY4NiIsCiAgICAgICJyZXNvdXJjZVByb3ZpZGVyIjoiTWljcm9zb2Z0LlJlc291cmNlcyIsCi AgICAgICJyZXNvdXJjZVVyaSI6Ii9zdWJzY3JpcHRpb25zL2VhOTg5NzRiLTVkMmEtNGQ5OC1hNzhhLTM4MmYzNzE1 ZDA3ZS9yZXNvdXJjZUdyb3Vwcy90ZXN0X2NvbnRhaW5lcl9tb2RlIiwKICAgICAgIm9wZXJhdGlvbk5hbWUiOiJNaW Nyb3NvZnQuUmVzb3VyY2VzL3N1YnNjcmlwdGlvbnMvcmVzb3VyY2VHcm91cHMvd3JpdGUiLAogICAgICAic3RhdHVz IjoiU3VjY2VlZGVkIiwKICAgfSwKICAgInRvcGljIjoiL3N1YnNjcmlwdGlvbnMvYWE5ODk3NGItNWQyYS00ZDk4LW E3OGEtMzgyZjM3MTVkMDdlIgp9""" q2 = QueueMessage() q2.id = 2 q2.dequeue_count = 0 q2.content = q1.content # Return 2 messages on first call, then none storage_mock.get_queue_messages.side_effect = [[q1, q2], []] host.poll_queue() self.assertEqual(2, run_policy_mock.call_count) run_policy_mock.reset_mock() # Return 5 messages on first call, then 2, then 0 storage_mock.get_queue_messages.side_effect = [[q1, q1, q1, q1, q1], [q1, q2], []] host.poll_queue() self.assertEqual(7, run_policy_mock.call_count) run_policy_mock.reset_mock() # High dequeue count q1.dequeue_count = 100 storage_mock.get_queue_messages.side_effect = [[q1, q2], []] host.poll_queue() self.assertEqual(1, run_policy_mock.call_count)
async def update_message( self, message, visibility_timeout=None, pop_receipt=None, # type: ignore content=None, timeout=None, **kwargs): # type: (Any, int, Optional[str], Optional[Any], Optional[int], Any) -> QueueMessage """Updates the visibility timeout of a message. You can also use this operation to update the contents of a message. This operation can be used to continually extend the invisibility of a queue message. This functionality can be useful if you want a worker role to "lease" a queue message. For example, if a worker role calls :func:`~receive_messages()` and recognizes that it needs more time to process a message, it can continually extend the message's invisibility until it is processed. If the worker role were to fail during processing, eventually the message would become visible again and another worker role could process it. If the key-encryption-key field is set on the local service object, this method will encrypt the content before uploading. :param str message: The message object or id identifying the message to update. :param int visibility_timeout: Specifies the new visibility timeout value, in seconds, relative to server time. The new value must be larger than or equal to 0, and cannot be larger than 7 days. The visibility timeout of a message cannot be set to a value later than the expiry time. A message can be updated until it has been deleted or has expired. The message object or message id identifying the message to update. :param str pop_receipt: A valid pop receipt value returned from an earlier call to the :func:`~receive_messages` or :func:`~update_message` operation. :param obj content: Message content. Allowed type is determined by the encode_function set on the service. Default is str. :param int timeout: The server timeout, expressed in seconds. :return: A :class:`~azure.storage.queue.models.QueueMessage` object. For convenience, this object is also populated with the content, although it is not returned by the service. :rtype: ~azure.storage.queue.models.QueueMessage Example: .. literalinclude:: ../tests/test_queue_samples_message.py :start-after: [START update_message] :end-before: [END update_message] :language: python :dedent: 12 :caption: Update a message. """ try: message_id = message.id message_text = content or message.content receipt = pop_receipt or message.pop_receipt insertion_time = message.insertion_time expiration_time = message.expiration_time dequeue_count = message.dequeue_count except AttributeError: message_id = message message_text = content receipt = pop_receipt insertion_time = None expiration_time = None dequeue_count = None if receipt is None: raise ValueError("pop_receipt must be present") if message_text is not None: self._config.message_encode_policy.configure( self.require_encryption, self.key_encryption_key, self.key_resolver_function) message_text = self._config.message_encode_policy(message_text) updated = GenQueueMessage(message_text=message_text) else: updated = None # type: ignore try: response = await self._client.message_id.update( queue_message=updated, visibilitytimeout=visibility_timeout or 0, timeout=timeout, pop_receipt=receipt, cls=return_response_headers, queue_message_id=message_id, **kwargs) new_message = QueueMessage(content=message_text) new_message.id = message_id new_message.insertion_time = insertion_time new_message.expiration_time = expiration_time new_message.dequeue_count = dequeue_count new_message.pop_receipt = response["popreceipt"] new_message.time_next_visible = response["time_next_visible"] return new_message except StorageErrorException as error: process_storage_error(error)
async def enqueue_message( # type: ignore self, content, # type: Any visibility_timeout=None, # type: Optional[int] time_to_live=None, # type: Optional[int] timeout=None, # type: Optional[int] **kwargs # type: Optional[Any] ): # type: (...) -> QueueMessage """Adds a new message to the back of the message queue. The visibility timeout specifies the time that the message will be invisible. After the timeout expires, the message will become visible. If a visibility timeout is not specified, the default value of 0 is used. The message time-to-live specifies how long a message will remain in the queue. The message will be deleted from the queue when the time-to-live period expires. If the key-encryption-key field is set on the local service object, this method will encrypt the content before uploading. :param obj content: Message content. Allowed type is determined by the encode_function set on the service. Default is str. The encoded message can be up to 64KB in size. :param int visibility_timeout: If not specified, the default value is 0. Specifies the new visibility timeout value, in seconds, relative to server time. The value must be larger than or equal to 0, and cannot be larger than 7 days. The visibility timeout of a message cannot be set to a value later than the expiry time. visibility_timeout should be set to a value smaller than the time-to-live value. :param int time_to_live: Specifies the time-to-live interval for the message, in seconds. The time-to-live may be any positive number or -1 for infinity. If this parameter is omitted, the default time-to-live is 7 days. :param int timeout: The server timeout, expressed in seconds. :return: A :class:`~azure.storage.queue.models.QueueMessage` object. This object is also populated with the content although it is not returned from the service. :rtype: ~azure.storage.queue.models.QueueMessage Example: .. literalinclude:: ../tests/test_queue_samples_message.py :start-after: [START enqueue_messages] :end-before: [END enqueue_messages] :language: python :dedent: 12 :caption: Enqueue messages. """ self._config.message_encode_policy.configure( self.require_encryption, self.key_encryption_key, self.key_resolver_function) content = self._config.message_encode_policy(content) new_message = GenQueueMessage(message_text=content) try: enqueued = await self._client.messages.enqueue( queue_message=new_message, visibilitytimeout=visibility_timeout, message_time_to_live=time_to_live, timeout=timeout, **kwargs) queue_message = QueueMessage(content=new_message.message_text) queue_message.id = enqueued[0].message_id queue_message.insertion_time = enqueued[0].insertion_time queue_message.expiration_time = enqueued[0].expiration_time queue_message.pop_receipt = enqueued[0].pop_receipt queue_message.time_next_visible = enqueued[0].time_next_visible return queue_message except StorageErrorException as error: process_storage_error(error)