def test_pop_unbalanced_queues(self): client = QueuedIngestClient("some-cluster") fake_receive = fake_receive_factory( lambda queue_name, messages_per_page=1: [mock_message(success=False) for _ in range(0, messages_per_page)] if "1" in queue_name else []) with mock.patch.object( client._resource_manager, "get_successful_ingestions_queues"), mock.patch.object( client._resource_manager, "get_failed_ingestions_queues" ) as mocked_get_failed_qs, mock.patch.object( QueueClient, "receive_messages", autospec=True, side_effect=fake_receive, ) as q_receive_mock, mock.patch.object(QueueClient, "delete_message", return_value=None): fake_failed_queue1 = _ResourceUri( "mocked_storage_account_f1", OBJECT_TYPE, "queue", "mocked_qf_1_name", ENDPOINT_SUFFIX, ) fake_failed_queue2 = _ResourceUri( "mocked_storage_account_f2", OBJECT_TYPE, "queue", "mocked_qf_2_name", ENDPOINT_SUFFIX, ) mocked_get_failed_qs.return_value = [ fake_failed_queue1, fake_failed_queue2 ] qs = KustoIngestStatusQueues(client) get_failure_actual = qs.failure.pop(6) assert len(get_failure_actual) == 6 for m in get_failure_actual: assert isinstance(m, FailureMessage) assert q_receive_mock.call_count == 3 actual = {} for call_args in q_receive_mock.call_args_list: actual[call_args[0][0].queue_name] = actual.get( call_args[0][0].queue_name, 0) + call_args[1]["messages_per_page"] assert actual[fake_failed_queue2.object_name] + actual[ fake_failed_queue1.object_name] == (4 + 4 + 6)
def test_isempty(self): client = QueuedIngestClient("some-cluster") fake_peek = fake_peek_factory(lambda queue_name, num_messages=1: [ mock_message(success=True) for _ in range(0, num_messages) ] if "qs" in queue_name else []) with mock.patch.object(client._resource_manager, "get_successful_ingestions_queues" ) as mocked_get_success_qs, mock.patch.object( client._resource_manager, "get_failed_ingestions_queues" ) as mocked_get_failed_qs, mock.patch.object( QueueClient, "peek_messages", autospec=True, side_effect=fake_peek) as q_mock: fake_failed_queue = _ResourceUri( "mocked_storage_account1", OBJECT_TYPE, "queue", "mocked_qf_name", ENDPOINT_SUFFIX, ) fake_success_queue = _ResourceUri( "mocked_storage_account2", OBJECT_TYPE, "queue", "mocked_qs_name", ENDPOINT_SUFFIX, ) mocked_get_success_qs.return_value = [fake_success_queue] mocked_get_failed_qs.return_value = [fake_failed_queue] qs = KustoIngestStatusQueues(client) assert qs.success.is_empty() is False assert qs.failure.is_empty() is True assert q_mock.call_count == 2 assert q_mock.call_args_list[0][1]["max_messages"] == 2 assert q_mock.call_args_list[1][1]["max_messages"] == 2
def test_isempty(self, mocked_q_peek_messages): client = KustoIngestClient("some-cluster") with mock.patch.object( client._resource_manager, "get_successful_ingestions_queues" ) as mocked_get_success_qs, mock.patch.object( client._resource_manager, "get_failed_ingestions_queues") as mocked_get_failed_qs: fake_failed_queue = _ResourceUri("mocked_storage_account1", "queue", "mocked_qf_name", "mocked_sas") fake_success_queue = _ResourceUri("mocked_storage_account2", "queue", "mocked_qs_name", "mocked_sas") mocked_get_success_qs.return_value = [fake_success_queue] mocked_get_failed_qs.return_value = [fake_failed_queue] mocked_q_peek_messages.side_effect = ( lambda queue_name, num_messages=1: [] if queue_name == fake_failed_queue.object_name else [QueueMessage() for _ in range(0, num_messages)]) qs = KustoIngestStatusQueues(client) assert qs.success.is_empty() == False assert qs.failure.is_empty() == True assert mocked_q_peek_messages.call_count == 2 assert mocked_q_peek_messages.call_args_list[0][0][ 0] == fake_success_queue.object_name assert mocked_q_peek_messages.call_args_list[0][1][ "num_messages"] == 2 assert mocked_q_peek_messages.call_args_list[1][0][ 0] == fake_failed_queue.object_name assert mocked_q_peek_messages.call_args_list[1][1][ "num_messages"] == 2
def test_pop(self): client = QueuedIngestClient("some-cluster") fake_receive = fake_receive_factory( lambda queue_name, num_messages=1: [ mock_message(success=True) if "qs" in queue_name else mock_message(success=False) for _ in range(0, num_messages) ]) with mock.patch.object(client._resource_manager, "get_successful_ingestions_queues" ) as mocked_get_success_qs, mock.patch.object( client._resource_manager, "get_failed_ingestions_queues" ) as mocked_get_failed_qs, mock.patch.object( QueueClient, "receive_messages", autospec=True, side_effect=fake_receive, ) as q_receive_mock, mock.patch.object( QueueClient, "delete_message", return_value=None) as q_del_mock: fake_failed_queue1 = _ResourceUri( "mocked_storage_account_f1", OBJECT_TYPE, "queue", "mocked_qf_1_name", ENDPOINT_SUFFIX, ) fake_failed_queue2 = _ResourceUri( "mocked_storage_account_f2", OBJECT_TYPE, "queue", "mocked_qf_2_name", ENDPOINT_SUFFIX, ) fake_success_queue = _ResourceUri( "mocked_storage_account2", OBJECT_TYPE, "queue", "mocked_qs_name", ENDPOINT_SUFFIX, ) mocked_get_success_qs.return_value = [fake_success_queue] mocked_get_failed_qs.return_value = [ fake_failed_queue1, fake_failed_queue2 ] qs = KustoIngestStatusQueues(client) get_success_actual = qs.success.pop() get_failure_actual = qs.failure.pop(6) assert len(get_success_actual) == 1 assert len(get_failure_actual) == 6 for m in get_failure_actual: assert isinstance(m, FailureMessage) for m in get_success_actual: assert isinstance(m, SuccessMessage) assert q_receive_mock.call_count == 3 assert q_del_mock.call_count == len(get_success_actual) + len( get_failure_actual) assert q_receive_mock.call_args_list[0][1][ "messages_per_page"] == 2 actual = { q_receive_mock.call_args_list[1][0][0].queue_name: q_receive_mock.call_args_list[1][1]["messages_per_page"], q_receive_mock.call_args_list[2][0][0].queue_name: q_receive_mock.call_args_list[2][1]["messages_per_page"], } assert actual[fake_failed_queue2.object_name] == 4 assert actual[fake_failed_queue1.object_name] == 4
def test_peek(self): client = QueuedIngestClient("some-cluster") fake_peek = fake_peek_factory(lambda queue_name, num_messages=1: [ mock_message(success=True) if "qs" in queue_name else mock_message( success=False) for _ in range(0, num_messages) ]) with mock.patch.object(client._resource_manager, "get_successful_ingestions_queues" ) as mocked_get_success_qs, mock.patch.object( client._resource_manager, "get_failed_ingestions_queues" ) as mocked_get_failed_qs, mock.patch.object( QueueClient, "peek_messages", autospec=True, side_effect=fake_peek) as q_mock: fake_failed_queue1 = _ResourceUri( "mocked_storage_account_f1", OBJECT_TYPE, "queue", "mocked_qf_1_name", ENDPOINT_SUFFIX, ) fake_failed_queue2 = _ResourceUri( "mocked_storage_account_f2", OBJECT_TYPE, "queue", "mocked_qf_2_name", ENDPOINT_SUFFIX, ) fake_success_queue = _ResourceUri( "mocked_storage_account2", OBJECT_TYPE, "queue", "mocked_qs_name", ENDPOINT_SUFFIX, ) mocked_get_success_qs.return_value = [fake_success_queue] mocked_get_failed_qs.return_value = [ fake_failed_queue1, fake_failed_queue2 ] qs = KustoIngestStatusQueues(client) peek_success_actual = qs.success.peek() peek_failure_actual = qs.failure.peek(6) assert len(peek_success_actual) == 1 for m in peek_failure_actual: assert isinstance(m, FailureMessage) is True for m in peek_success_actual: assert isinstance(m, SuccessMessage) is True assert len(peek_failure_actual) == 6 actual = {} assert len(QueueClient.peek_messages.call_args_list) == 3 for call_args in q_mock.call_args_list: actual[call_args[0][0].queue_name] = actual.get( call_args[0][0].queue_name, 0) + call_args[1]["max_messages"] assert actual[fake_failed_queue2.object_name] == 4 assert actual[fake_failed_queue1.object_name] == 4 assert actual[fake_success_queue.object_name] == 2
def test_peek(self, mocked_q_peek_messages): client = KustoIngestClient("some-cluster") with mock.patch.object( client._resource_manager, "get_successful_ingestions_queues" ) as mocked_get_success_qs, mock.patch.object( client._resource_manager, "get_failed_ingestions_queues") as mocked_get_failed_qs: fake_failed_queue1 = _ResourceUri("mocked_storage_account_f1", "queue", "mocked_qf_1_name", "mocked_sas") fake_failed_queue2 = _ResourceUri("mocked_storage_account_f2", "queue", "mocked_qf_2_name", "mocked_sas") fake_success_queue = _ResourceUri("mocked_storage_account2", "queue", "mocked_qs_name", "mocked_sas") mocked_get_success_qs.return_value = [fake_success_queue] mocked_get_failed_qs.return_value = [ fake_failed_queue1, fake_failed_queue2 ] def mock_message(success): m = QueueMessage() m.id = uuid4() m.insertion_time = time.time() m.expiration_time = None m.dequeue_count = None if success: content = { "OperationId": str(m.id), "Database": "db1", "Table": "table1", "IngestionSourceId": str(m.id), "IngestionSourcePath": "blob/path", "RootActivityId": "1", "SucceededOn": time.time(), } else: content = { "OperationId": str(m.id), "Database": "db1", "Table": "table1", "IngestionSourceId": str(m.id), "IngestionSourcePath": "blob/path", "RootActivityId": "1", "FailedOn": time.time(), "Details": "", "ErrorCode": "1", "FailureStatus": "", "OriginatesFromUpdatePolicy": "", "ShouldRetry": False, } m.content = str( base64.b64encode( json.dumps(content).encode("utf-8")).decode("utf-8")) m.pop_receipt = None m.time_next_visible = None return m mocked_q_peek_messages.side_effect = lambda queue_name, num_messages=1: [ mock_message(success=True) if queue_name in [fake_success_queue.object_name] else mock_message(success=False) for i in range(0, num_messages) ] qs = KustoIngestStatusQueues(client) peek_success_actual = qs.success.peek() peek_failure_actual = qs.failure.peek(6) assert len(peek_success_actual) == 1 for m in peek_failure_actual: assert isinstance(m, FailureMessage) == True for m in peek_success_actual: assert isinstance(m, SuccessMessage) == True assert len(peek_failure_actual) == 6 actual = {} assert len(mocked_q_peek_messages.call_args_list) == 3 for call_args in mocked_q_peek_messages.call_args_list: actual[call_args[0][0]] = actual.get( call_args[0][0], 0) + call_args[1]["num_messages"] assert actual[fake_failed_queue2.object_name] == 4 assert actual[fake_failed_queue1.object_name] == 4 assert actual[fake_success_queue.object_name] == 2