Beispiel #1
0
    def test_run_policy_for_event(self, add_job_mock, _0, _1, _2, _3):
        host = Host(DEFAULT_EVENT_QUEUE_ID, DEFAULT_EVENT_QUEUE_NAME,
                    DEFAULT_POLICY_STORAGE)

        host.policies = {
            'one': {
                'policy':
                ContainerHostTest.get_mock_policy({
                    'name': 'one',
                    'mode': {
                        'type': 'container-event',
                        'events': ['ResourceGroupWrite', 'VnetWrite']
                    }
                })
            }
        }

        message = QueueMessage()
        message.id = 1
        message.dequeue_count = 0
        message.content = \
            """eyAgCiAgICJzdWJqZWN0IjoiL3N1YnNjcmlwdGlvbnMvZWE5ODk3NGItNWQyYS00ZDk4LWE3OGEt
            MzgyZjM3MTVkMDdlL3Jlc291cmNlR3JvdXBzL3Rlc3RfY29udGFpbmVyX21vZGUiLAogICAiZXZl
            bnRUeXBlIjoiTWljcm9zb2Z0LlJlc291cmNlcy5SZXNvdXJjZVdyaXRlU3VjY2VzcyIsCiAgICJl
            dmVudFRpbWUiOiIyMDE5LTA3LTE2VDE4OjMwOjQzLjM1OTUyNTVaIiwKICAgImlkIjoiNjE5ZDI2
            NzQtYjM5Ni00MzU2LTk2MTktNmM1YTUyZmU0ZTg4IiwKICAgImRhdGEiOnsgICAgICAgIAogICAg
            ICAiY29ycmVsYXRpb25JZCI6IjdkZDVhNDc2LWUwNTItNDBlMi05OWU0LWJiOTg1MmRjMWY4NiIs
            CiAgICAgICJyZXNvdXJjZVByb3ZpZGVyIjoiTWljcm9zb2Z0LlJlc291cmNlcyIsCiAgICAgICJy
            ZXNvdXJjZVVyaSI6Ii9zdWJzY3JpcHRpb25zL2VhOTg5NzRiLTVkMmEtNGQ5OC1hNzhhLTM4MmYz
            NzE1ZDA3ZS9yZXNvdXJjZUdyb3Vwcy90ZXN0X2NvbnRhaW5lcl9tb2RlIiwKICAgICAgIm9wZXJh
            dGlvbk5hbWUiOiJNaWNyb3NvZnQuUmVzb3VyY2VzL3N1YnNjcmlwdGlvbnMvcmVzb3VyY2VHcm91
            cHMvd3JpdGUiLAogICAgICAic3RhdHVzIjoiU3VjY2VlZGVkIgogICB9LAogICAidG9waWMiOiIv
            c3Vic2NyaXB0aW9ucy9hYTk4OTc0Yi01ZDJhLTRkOTgtYTc4YS0zODJmMzcxNWQwN2UiCn0="""

        # run with real match
        host.run_policies_for_event(message)
        add_job_mock.assert_called_with(
            ANY,
            id='one619d2674-b396-4356-9619-6c5a52fe4e88',
            name='one',
            args=ANY,
            misfire_grace_time=ANY)

        add_job_mock.reset_mock()

        # run with no match
        host.policies = {}
        host.run_policies_for_event(message)
        self.assertFalse(add_job_mock.called)
    async def peek_messages(self,
                            max_messages=None,
                            timeout=None,
                            **kwargs):  # type: ignore
        # type: (Optional[int], Optional[int], Optional[Any]) -> List[QueueMessage]
        """Retrieves one or more messages from the front of the queue, but does
        not alter the visibility of the message.

        Only messages that are visible may be retrieved. When a message is retrieved
        for the first time with a call to :func:`~receive_messages`, its dequeue_count property
        is set to 1. If it is not deleted and is subsequently retrieved again, the
        dequeue_count property is incremented. The client may use this value to
        determine how many times a message has been retrieved. Note that a call
        to peek_messages does not increment the value of DequeueCount, but returns
        this value for the client to read.

        If the key-encryption-key or resolver field is set on the local service object,
        the messages will be decrypted before being returned.

        :param int max_messages:
            A nonzero integer value that specifies the number of
            messages to peek from the queue, up to a maximum of 32. By default,
            a single message is peeked from the queue with this operation.
        :param int timeout:
            The server timeout, expressed in seconds.
        :return:
            A list of :class:`~azure.storage.queue.QueueMessage` objects. Note that
            time_next_visible and pop_receipt will not be populated as peek does
            not pop the message and can only retrieve already visible messages.
        :rtype: list(:class:`~azure.storage.queue.QueueMessage`)

        .. admonition:: Example:

            .. literalinclude:: ../tests/test_queue_samples_message_async.py
                :start-after: [START async_peek_message]
                :end-before: [END async_peek_message]
                :language: python
                :dedent: 12
                :caption: Peek messages.
        """
        if max_messages and not 1 <= max_messages <= 32:
            raise ValueError(
                "Number of messages to peek should be between 1 and 32")
        self._config.message_decode_policy.configure(
            require_encryption=self.require_encryption,
            key_encryption_key=self.key_encryption_key,
            resolver=self.key_resolver_function)
        try:
            messages = await self._client.messages.peek(
                number_of_messages=max_messages,
                timeout=timeout,
                cls=self._config.message_decode_policy,
                **kwargs)
            wrapped_messages = []
            for peeked in messages:
                wrapped_messages.append(QueueMessage._from_generated(peeked))  # pylint: disable=protected-access
            return wrapped_messages
        except StorageErrorException as error:
            process_storage_error(error)
            def mock_message(success):
                m = QueueMessage()
                m.id = uuid4()
                m.insertion_time = time.time()
                m.expiration_time = None
                m.dequeue_count = None

                if success:
                    content = {
                        "OperationId": str(m.id),
                        "Database": "db1",
                        "Table": "table1",
                        "IngestionSourceId": str(m.id),
                        "IngestionSourcePath": "blob/path",
                        "RootActivityId": "1",
                        "SucceededOn": time.time(),
                    }
                else:
                    content = {
                        "OperationId": str(m.id),
                        "Database": "db1",
                        "Table": "table1",
                        "IngestionSourceId": str(m.id),
                        "IngestionSourcePath": "blob/path",
                        "RootActivityId": "1",
                        "FailedOn": time.time(),
                        "Details": "",
                        "ErrorCode": "1",
                        "FailureStatus": "",
                        "OriginatesFromUpdatePolicy": "",
                        "ShouldRetry": False,
                    }

                m.content = str(
                    base64.b64encode(
                        json.dumps(content).encode("utf-8")).decode("utf-8"))
                m.pop_receipt = None
                m.time_next_visible = None

                return m
    def test_isempty(self, mocked_q_peek_messages):
        client = KustoIngestClient("some-cluster")
        with mock.patch.object(
                client._resource_manager, "get_successful_ingestions_queues"
        ) as mocked_get_success_qs, mock.patch.object(
                client._resource_manager,
                "get_failed_ingestions_queues") as mocked_get_failed_qs:

            fake_failed_queue = _ResourceUri("mocked_storage_account1",
                                             "queue", "mocked_qf_name",
                                             "mocked_sas")
            fake_success_queue = _ResourceUri("mocked_storage_account2",
                                              "queue", "mocked_qs_name",
                                              "mocked_sas")

            mocked_get_success_qs.return_value = [fake_success_queue]
            mocked_get_failed_qs.return_value = [fake_failed_queue]

            mocked_q_peek_messages.side_effect = (
                lambda queue_name, num_messages=1: []
                if queue_name == fake_failed_queue.object_name else
                [QueueMessage() for _ in range(0, num_messages)])

            qs = KustoIngestStatusQueues(client)

            assert qs.success.is_empty() == False
            assert qs.failure.is_empty() == True

            assert mocked_q_peek_messages.call_count == 2
            assert mocked_q_peek_messages.call_args_list[0][0][
                0] == fake_success_queue.object_name
            assert mocked_q_peek_messages.call_args_list[0][1][
                "num_messages"] == 2

            assert mocked_q_peek_messages.call_args_list[1][0][
                0] == fake_failed_queue.object_name
            assert mocked_q_peek_messages.call_args_list[1][1][
                "num_messages"] == 2
Beispiel #5
0
    def test_poll_queue(self, run_policy_mock, storage_mock, _1, _2, _3):
        host = Host(DEFAULT_EVENT_QUEUE_ID, DEFAULT_EVENT_QUEUE_NAME,
                    DEFAULT_POLICY_STORAGE)

        host.policies = {
            'one': {
                'policy':
                ContainerHostTest.get_mock_policy({
                    'name': 'one',
                    'mode': {
                        'type': 'container-event',
                        'events': ['ResourceGroupWrite', 'VnetWrite']
                    }
                })
            }
        }

        q1 = QueueMessage()
        q1.id = 1
        q1.dequeue_count = 0
        q1.content = """eyAgCiAgICJzdWJqZWN0IjoiL3N1YnNjcmlwdGlvbnMvZWE5ODk3NGItNWQyYS00ZDk4LWE3
        OGEtMzgyZjM3MTVkMDdlL3Jlc291cmNlR3JvdXBzL3Rlc3RfY29udGFpbmVyX21vZGUiLAogICAiZXZlbnRUeXBlIj
        oiTWljcm9zb2Z0LlJlc291cmNlcy5SZXNvdXJjZVdyaXRlU3VjY2VzcyIsCiAgICJldmVudFRpbWUiOiIyMDE5LTA3
        LTE2VDE4OjMwOjQzLjM1OTUyNTVaIiwKICAgImlkIjoiNjE5ZDI2NzQtYjM5Ni00MzU2LTk2MTktNmM1YTUyZmU0ZT
        g4IiwKICAgImRhdGEiOnsgICAgICAgIAogICAgICAiY29ycmVsYXRpb25JZCI6IjdkZDVhNDc2LWUwNTItNDBlMi05
        OWU0LWJiOTg1MmRjMWY4NiIsCiAgICAgICJyZXNvdXJjZVByb3ZpZGVyIjoiTWljcm9zb2Z0LlJlc291cmNlcyIsCi
        AgICAgICJyZXNvdXJjZVVyaSI6Ii9zdWJzY3JpcHRpb25zL2VhOTg5NzRiLTVkMmEtNGQ5OC1hNzhhLTM4MmYzNzE1
        ZDA3ZS9yZXNvdXJjZUdyb3Vwcy90ZXN0X2NvbnRhaW5lcl9tb2RlIiwKICAgICAgIm9wZXJhdGlvbk5hbWUiOiJNaW
        Nyb3NvZnQuUmVzb3VyY2VzL3N1YnNjcmlwdGlvbnMvcmVzb3VyY2VHcm91cHMvd3JpdGUiLAogICAgICAic3RhdHVz
        IjoiU3VjY2VlZGVkIiwKICAgfSwKICAgInRvcGljIjoiL3N1YnNjcmlwdGlvbnMvYWE5ODk3NGItNWQyYS00ZDk4LW
        E3OGEtMzgyZjM3MTVkMDdlIgp9"""

        q2 = QueueMessage()
        q2.id = 2
        q2.dequeue_count = 0
        q2.content = q1.content

        # Return 2 messages on first call, then none
        storage_mock.get_queue_messages.side_effect = [[q1, q2], []]
        host.poll_queue()
        self.assertEqual(2, run_policy_mock.call_count)
        run_policy_mock.reset_mock()

        # Return 5 messages on first call, then 2, then 0
        storage_mock.get_queue_messages.side_effect = [[q1, q1, q1, q1, q1],
                                                       [q1, q2], []]
        host.poll_queue()
        self.assertEqual(7, run_policy_mock.call_count)
        run_policy_mock.reset_mock()

        # High dequeue count
        q1.dequeue_count = 100
        storage_mock.get_queue_messages.side_effect = [[q1, q2], []]
        host.poll_queue()
        self.assertEqual(1, run_policy_mock.call_count)
Beispiel #6
0
    async def update_message(
            self,
            message,
            visibility_timeout=None,
            pop_receipt=None,  # type: ignore
            content=None,
            timeout=None,
            **kwargs):
        # type: (Any, int, Optional[str], Optional[Any], Optional[int], Any) -> QueueMessage
        """Updates the visibility timeout of a message. You can also use this
        operation to update the contents of a message.

        This operation can be used to continually extend the invisibility of a
        queue message. This functionality can be useful if you want a worker role
        to "lease" a queue message. For example, if a worker role calls :func:`~receive_messages()`
        and recognizes that it needs more time to process a message, it can
        continually extend the message's invisibility until it is processed. If
        the worker role were to fail during processing, eventually the message
        would become visible again and another worker role could process it.

        If the key-encryption-key field is set on the local service object, this method will
        encrypt the content before uploading.

        :param str message:
            The message object or id identifying the message to update.
        :param int visibility_timeout:
            Specifies the new visibility timeout value, in seconds,
            relative to server time. The new value must be larger than or equal
            to 0, and cannot be larger than 7 days. The visibility timeout of a
            message cannot be set to a value later than the expiry time. A
            message can be updated until it has been deleted or has expired.
            The message object or message id identifying the message to update.
        :param str pop_receipt:
            A valid pop receipt value returned from an earlier call
            to the :func:`~receive_messages` or :func:`~update_message` operation.
        :param obj content:
            Message content. Allowed type is determined by the encode_function
            set on the service. Default is str.
        :param int timeout:
            The server timeout, expressed in seconds.
        :return:
            A :class:`~azure.storage.queue.models.QueueMessage` object. For convenience,
            this object is also populated with the content, although it is not returned by the service.
        :rtype: ~azure.storage.queue.models.QueueMessage

        Example:
            .. literalinclude:: ../tests/test_queue_samples_message.py
                :start-after: [START update_message]
                :end-before: [END update_message]
                :language: python
                :dedent: 12
                :caption: Update a message.
        """
        try:
            message_id = message.id
            message_text = content or message.content
            receipt = pop_receipt or message.pop_receipt
            insertion_time = message.insertion_time
            expiration_time = message.expiration_time
            dequeue_count = message.dequeue_count
        except AttributeError:
            message_id = message
            message_text = content
            receipt = pop_receipt
            insertion_time = None
            expiration_time = None
            dequeue_count = None

        if receipt is None:
            raise ValueError("pop_receipt must be present")
        if message_text is not None:
            self._config.message_encode_policy.configure(
                self.require_encryption, self.key_encryption_key,
                self.key_resolver_function)
            message_text = self._config.message_encode_policy(message_text)
            updated = GenQueueMessage(message_text=message_text)
        else:
            updated = None  # type: ignore
        try:
            response = await self._client.message_id.update(
                queue_message=updated,
                visibilitytimeout=visibility_timeout or 0,
                timeout=timeout,
                pop_receipt=receipt,
                cls=return_response_headers,
                queue_message_id=message_id,
                **kwargs)
            new_message = QueueMessage(content=message_text)
            new_message.id = message_id
            new_message.insertion_time = insertion_time
            new_message.expiration_time = expiration_time
            new_message.dequeue_count = dequeue_count
            new_message.pop_receipt = response["popreceipt"]
            new_message.time_next_visible = response["time_next_visible"]
            return new_message
        except StorageErrorException as error:
            process_storage_error(error)
Beispiel #7
0
    async def enqueue_message(  # type: ignore
            self,
            content,  # type: Any
            visibility_timeout=None,  # type: Optional[int]
            time_to_live=None,  # type: Optional[int]
            timeout=None,  # type: Optional[int]
            **kwargs  # type: Optional[Any]
    ):
        # type: (...) -> QueueMessage
        """Adds a new message to the back of the message queue.

        The visibility timeout specifies the time that the message will be
        invisible. After the timeout expires, the message will become visible.
        If a visibility timeout is not specified, the default value of 0 is used.

        The message time-to-live specifies how long a message will remain in the
        queue. The message will be deleted from the queue when the time-to-live
        period expires.

        If the key-encryption-key field is set on the local service object, this method will
        encrypt the content before uploading.

        :param obj content:
            Message content. Allowed type is determined by the encode_function
            set on the service. Default is str. The encoded message can be up to
            64KB in size.
        :param int visibility_timeout:
            If not specified, the default value is 0. Specifies the
            new visibility timeout value, in seconds, relative to server time.
            The value must be larger than or equal to 0, and cannot be
            larger than 7 days. The visibility timeout of a message cannot be
            set to a value later than the expiry time. visibility_timeout
            should be set to a value smaller than the time-to-live value.
        :param int time_to_live:
            Specifies the time-to-live interval for the message, in
            seconds. The time-to-live may be any positive number or -1 for infinity. If this
            parameter is omitted, the default time-to-live is 7 days.
        :param int timeout:
            The server timeout, expressed in seconds.
        :return:
            A :class:`~azure.storage.queue.models.QueueMessage` object.
            This object is also populated with the content although it is not
            returned from the service.
        :rtype: ~azure.storage.queue.models.QueueMessage

        Example:
            .. literalinclude:: ../tests/test_queue_samples_message.py
                :start-after: [START enqueue_messages]
                :end-before: [END enqueue_messages]
                :language: python
                :dedent: 12
                :caption: Enqueue messages.
        """
        self._config.message_encode_policy.configure(
            self.require_encryption, self.key_encryption_key,
            self.key_resolver_function)
        content = self._config.message_encode_policy(content)
        new_message = GenQueueMessage(message_text=content)

        try:
            enqueued = await self._client.messages.enqueue(
                queue_message=new_message,
                visibilitytimeout=visibility_timeout,
                message_time_to_live=time_to_live,
                timeout=timeout,
                **kwargs)
            queue_message = QueueMessage(content=new_message.message_text)
            queue_message.id = enqueued[0].message_id
            queue_message.insertion_time = enqueued[0].insertion_time
            queue_message.expiration_time = enqueued[0].expiration_time
            queue_message.pop_receipt = enqueued[0].pop_receipt
            queue_message.time_next_visible = enqueued[0].time_next_visible
            return queue_message
        except StorageErrorException as error:
            process_storage_error(error)