def test_sb_client_bad_credentials(live_servicebus_config, standard_queue): client = ServiceBusClient( service_namespace=live_servicebus_config['hostname'], shared_access_key_name="invalid", shared_access_key_value="invalid", debug=True) with pytest.raises(AzureHttpError): client.get_queue(standard_queue)
def test_sb_client_bad_namespace(live_servicebus_config): client = ServiceBusClient( service_namespace="invalid", shared_access_key_name="invalid", shared_access_key_value="invalid", debug=True) with pytest.raises(ServiceBusConnectionError): client.get_queue("testq")
def test_sb_client_bad_entity(live_servicebus_config): client = ServiceBusClient( service_namespace=live_servicebus_config['hostname'], shared_access_key_name=live_servicebus_config['key_name'], shared_access_key_value=live_servicebus_config['access_key'], debug=True) with pytest.raises(ServiceBusResourceNotFound): client.get_queue("invalid") with pytest.raises(ServiceBusResourceNotFound): client.get_topic("invalid")
def test_qsession_by_servicebus_client_browse_messages_client(live_servicebus_config, partitioned_session_queue): client = ServiceBusClient( service_namespace=live_servicebus_config['hostname'], shared_access_key_name=live_servicebus_config['key_name'], shared_access_key_value=live_servicebus_config['access_key'], debug=True) queue_client = client.get_queue(partitioned_session_queue) session_id = str(uuid.uuid4()) with queue_client.get_sender(session=session_id) as sender: for i in range(5): message = Message("Test message no. {}".format(i)) message.partition_key = "MyPartitionKey" sender.send(message) with pytest.raises(ValueError): messages = queue_client.peek(5) messages = queue_client.peek(5, session=session_id) assert len(messages) == 5 assert all(isinstance(m, PeekMessage) for m in messages) for message in messages: print_message(message) with pytest.raises(TypeError): message.complete()
def test_queue_message_batch(live_servicebus_config, standard_queue): client = ServiceBusClient( service_namespace=live_servicebus_config['hostname'], shared_access_key_name=live_servicebus_config['key_name'], shared_access_key_value=live_servicebus_config['access_key'], debug=True) queue_client = client.get_queue(standard_queue) def message_content(): for i in range(5): yield "Message no. {}".format(i) with queue_client.get_sender() as sender: message = BatchMessage(message_content()) sender.send(message) with queue_client.get_receiver() as receiver: messages = receiver.fetch_next(timeout=10) recv = True while recv: recv = receiver.fetch_next(timeout=10) messages.extend(recv) assert len(messages) == 5 for m in messages: print_message(m) m.complete()
def test_queue_message_lock_renew(live_servicebus_config, standard_queue): client = ServiceBusClient( service_namespace=live_servicebus_config['hostname'], shared_access_key_name=live_servicebus_config['key_name'], shared_access_key_value=live_servicebus_config['access_key'], debug=True) import uuid queue_client = client.get_queue(standard_queue) with queue_client.get_sender() as sender: content = str(uuid.uuid4()) message = Message(content) sender.send(message) with queue_client.get_receiver() as receiver: messages = receiver.fetch_next(timeout=10) assert len(messages) == 1 time.sleep(15) messages[0].renew_lock() time.sleep(15) messages[0].renew_lock() time.sleep(15) assert not messages[0].expired messages[0].complete() with queue_client.get_receiver() as receiver: messages = receiver.fetch_next(timeout=10) assert len(messages) == 0
def test_session_by_servicebus_client_browse_messages_with_receiver( live_servicebus_config, session_queue): client = ServiceBusClient( service_namespace=live_servicebus_config['hostname'], shared_access_key_name=live_servicebus_config['key_name'], shared_access_key_value=live_servicebus_config['access_key'], debug=False) queue_client = client.get_queue(session_queue) session_id = str(uuid.uuid4()) with queue_client.get_receiver(idle_timeout=5, session=session_id) as receiver: with queue_client.get_sender(session=session_id) as sender: for i in range(5): message = Message("Test message no. {}".format(i)) sender.send(message) messages = receiver.peek(5) assert len(messages) > 0 assert all(isinstance(m, PeekMessage) for m in messages) for message in messages: print_message(message) with pytest.raises(TypeError): message.complete()
def test_session_cancel_scheduled_messages(live_servicebus_config, session_queue): client = ServiceBusClient( service_namespace=live_servicebus_config['hostname'], shared_access_key_name=live_servicebus_config['key_name'], shared_access_key_value=live_servicebus_config['access_key'], debug=True) session_id = str(uuid.uuid4()) queue_client = client.get_queue(session_queue) enqueue_time = (datetime.now() + timedelta(minutes=2)).replace(microsecond=0) with queue_client.get_sender(session=session_id) as sender: message_a = Message("Test scheduled message") message_b = Message("Test scheduled message") tokens = sender.schedule(enqueue_time, message_a, message_b) assert len(tokens) == 2 sender.cancel_scheduled_messages(*tokens) with queue_client.get_receiver(session=session_id) as receiver: messages = [] count = 0 while not messages and count < 13: messages = receiver.fetch_next(timeout=10) receiver.renew_lock() count += 1 assert len(messages) == 0
def test_queue_by_servicebus_client_fail_send_messages(live_servicebus_config, standard_queue): client = ServiceBusClient( service_namespace=live_servicebus_config['hostname'], shared_access_key_name=live_servicebus_config['key_name'], shared_access_key_value=live_servicebus_config['access_key'], debug=True) queue_client = client.get_queue(standard_queue) too_large = "A" * 1024 * 512 try: results = queue_client.send(Message(too_large)) except MessageSendFailed: pytest.skip("Open issue for uAMQP on OSX") assert len(results) == 1 assert not results[0][0] assert isinstance(results[0][1], MessageSendFailed) with queue_client.get_sender() as sender: with pytest.raises(MessageSendFailed): sender.send(Message(too_large)) with queue_client.get_sender() as sender: sender.queue_message(Message(too_large)) results = sender.send_pending_messages() assert len(results) == 1 assert not results[0][0] assert isinstance(results[0][1], MessageSendFailed)
def test_session_message_expiry(live_servicebus_config, session_queue): client = ServiceBusClient( service_namespace=live_servicebus_config['hostname'], shared_access_key_name=live_servicebus_config['key_name'], shared_access_key_value=live_servicebus_config['access_key'], debug=True) session_id = str(uuid.uuid4()) queue_client = client.get_queue(session_queue) with queue_client.get_sender() as sender: message = Message("Testing expired messages") message.session_id = session_id sender.send(message) with queue_client.get_receiver(session=session_id) as receiver: messages = receiver.fetch_next(timeout=10) assert len(messages) == 1 print_message(messages[0]) time.sleep(30) with pytest.raises(TypeError): messages[0].expired with pytest.raises(TypeError): messages[0].renew_lock() assert receiver.expired with pytest.raises(SessionLockExpired): messages[0].complete() with pytest.raises(SessionLockExpired): receiver.renew_lock() with queue_client.get_receiver(session=session_id) as receiver: messages = receiver.fetch_next(timeout=30) assert len(messages) == 1 print_message(messages[0]) #assert messages[0].header.delivery_count # TODO confirm this with service messages[0].complete()
def test_queue_by_servicebus_client_iter_messages_with_defer( live_servicebus_config, standard_queue): client = ServiceBusClient( service_namespace=live_servicebus_config['hostname'], shared_access_key_name=live_servicebus_config['key_name'], shared_access_key_value=live_servicebus_config['access_key'], debug=True) queue_client = client.get_queue(standard_queue) deferred_messages = [] with queue_client.get_receiver( idle_timeout=5, mode=ReceiveSettleMode.PeekLock) as receiver: with queue_client.get_sender() as sender: for i in range(10): message = Message("Deferred message no. {}".format(i)) sender.send(message) count = 0 for message in receiver: deferred_messages.append(message.sequence_number) print_message(message) count += 1 message.defer() assert count == 10 with queue_client.get_receiver( idle_timeout=5, mode=ReceiveSettleMode.PeekLock) as receiver: count = 0 for message in receiver: print_message(message) message.complete() count += 1 assert count == 0
def test_queue_by_servicebus_client_iter_messages_with_retrieve_deferred_not_found( live_servicebus_config, standard_queue): client = ServiceBusClient( service_namespace=live_servicebus_config['hostname'], shared_access_key_name=live_servicebus_config['key_name'], shared_access_key_value=live_servicebus_config['access_key'], debug=True) queue_client = client.get_queue(standard_queue) deferred_messages = [] with queue_client.get_receiver( idle_timeout=5, mode=ReceiveSettleMode.PeekLock) as receiver: with queue_client.get_sender() as sender: for i in range(3): message = Message("Deferred message no. {}".format(i)) sender.send(message) count = 0 for message in receiver: deferred_messages.append(message.sequence_number) print_message(message) count += 1 message.defer() assert count == 3 with pytest.raises(ServiceBusError): deferred = queue_client.receive_deferred_messages( [3, 4], mode=ReceiveSettleMode.PeekLock) with pytest.raises(ServiceBusError): deferred = queue_client.receive_deferred_messages( [5, 6, 7], mode=ReceiveSettleMode.PeekLock)
def test_queue_by_servicebus_client_iter_messages_simple( live_servicebus_config, standard_queue): client = ServiceBusClient( service_namespace=live_servicebus_config['hostname'], shared_access_key_name=live_servicebus_config['key_name'], shared_access_key_value=live_servicebus_config['access_key'], debug=True) queue_client = client.get_queue(standard_queue) with queue_client.get_receiver( idle_timeout=5, mode=ReceiveSettleMode.PeekLock) as receiver: with queue_client.get_sender() as sender: for i in range(10): message = Message("Iter message no. {}".format(i)) sender.send(message) count = 0 for message in receiver: print_message(message) message.complete() with pytest.raises(MessageAlreadySettled): message.complete() with pytest.raises(MessageAlreadySettled): message.renew_lock() count += 1 with pytest.raises(InvalidHandlerState): next(receiver) assert count == 10
def test_queue_cancel_scheduled_messages(live_servicebus_config, standard_queue): client = ServiceBusClient( service_namespace=live_servicebus_config['hostname'], shared_access_key_name=live_servicebus_config['key_name'], shared_access_key_value=live_servicebus_config['access_key'], debug=True) queue_client = client.get_queue(standard_queue) enqueue_time = (datetime.utcnow() + timedelta(minutes=2)).replace(microsecond=0) with queue_client.get_receiver() as receiver: with queue_client.get_sender() as sender: message_a = Message("Test scheduled message") message_b = Message("Test scheduled message") tokens = sender.schedule(enqueue_time, message_a, message_b) assert len(tokens) == 2 sender.cancel_scheduled_messages(*tokens) messages = receiver.fetch_next(timeout=120) try: assert len(messages) == 0 except AssertionError: for m in messages: print(str(m)) m.complete() raise
def test_queue_schedule_message(live_servicebus_config, standard_queue): client = ServiceBusClient( service_namespace=live_servicebus_config['hostname'], shared_access_key_name=live_servicebus_config['key_name'], shared_access_key_value=live_servicebus_config['access_key'], debug=True) import uuid queue_client = client.get_queue(standard_queue) enqueue_time = (datetime.utcnow() + timedelta(minutes=2)).replace(microsecond=0) with queue_client.get_receiver() as receiver: with queue_client.get_sender() as sender: content = str(uuid.uuid4()) message_id = uuid.uuid4() message = Message(content) message.properties.message_id = message_id message.schedule(enqueue_time) sender.send(message) messages = receiver.fetch_next(timeout=120) if messages: try: data = str(messages[0]) assert data == content assert messages[0].properties.message_id == message_id assert messages[0].scheduled_enqueue_time == enqueue_time assert messages[0].scheduled_enqueue_time == messages[ 0].enqueued_time.replace(microsecond=0) assert len(messages) == 1 finally: for m in messages: m.complete() else: raise Exception("Failed to receive schdeduled message.")
def test_session_by_servicebus_client_iter_messages_with_retrieve_deferred_receiver_deletemode(live_servicebus_config, session_queue): client = ServiceBusClient( service_namespace=live_servicebus_config['hostname'], shared_access_key_name=live_servicebus_config['key_name'], shared_access_key_value=live_servicebus_config['access_key'], debug=True) queue_client = client.get_queue(session_queue) deferred_messages = [] session_id = str(uuid.uuid4()) messages = [Message("Deferred message no. {}".format(i)) for i in range(10)] results = queue_client.send(messages, session=session_id) assert all(result[0] for result in results) count = 0 session = queue_client.get_receiver(session=session_id, idle_timeout=5) for message in session: deferred_messages.append(message.sequence_number) print_message(message) count += 1 message.defer() assert count == 10 with queue_client.get_receiver(session=session_id, idle_timeout=5) as session: deferred = session.receive_deferred_messages(deferred_messages, mode=ReceiveSettleMode.ReceiveAndDelete) assert len(deferred) == 10 for message in deferred: assert isinstance(message, DeferredMessage) with pytest.raises(MessageAlreadySettled): message.complete() with pytest.raises(ServiceBusError): deferred = session.receive_deferred_messages(deferred_messages)
def test_queue_by_servicebus_client_fail_send_batch_messages( live_servicebus_config, standard_queue): pytest.skip("TODO: Pending bugfix in uAMQP") def batch_data(): for i in range(3): yield str(i) * 1024 * 256 client = ServiceBusClient( service_namespace=live_servicebus_config['hostname'], shared_access_key_name=live_servicebus_config['key_name'], shared_access_key_value=live_servicebus_config['access_key'], debug=True) queue_client = client.get_queue(standard_queue) results = queue_client.send(BatchMessage(batch_data())) assert len(results) == 4 assert not results[0][0] assert isinstance(results[0][1], MessageSendFailed) with queue_client.get_sender() as sender: with pytest.raises(MessageSendFailed): sender.send(BatchMessage(batch_data())) with queue_client.get_sender() as sender: sender.queue_message(BatchMessage(batch_data())) results = sender.send_pending_messages() assert len(results) == 4 assert not results[0][0] assert isinstance(results[0][1], MessageSendFailed)
def test_session_by_servicebus_client_iter_messages_with_retrieve_deferred_client(live_servicebus_config, session_queue): client = ServiceBusClient( service_namespace=live_servicebus_config['hostname'], shared_access_key_name=live_servicebus_config['key_name'], shared_access_key_value=live_servicebus_config['access_key'], debug=True) queue_client = client.get_queue(session_queue) deferred_messages = [] session_id = str(uuid.uuid4()) with queue_client.get_sender(session=session_id) as sender: for i in range(10): message = Message("Deferred message no. {}".format(i)) sender.send(message) session = queue_client.get_receiver(session=session_id, idle_timeout=5) count = 0 for message in session: deferred_messages.append(message.sequence_number) print_message(message) count += 1 message.defer() assert count == 10 with pytest.raises(ValueError): deferred = queue_client.receive_deferred_messages(deferred_messages, session=session_id) with pytest.raises(ValueError): queue_client.settle_deferred_messages('completed', [message], session=session_id)
def test_queue_message_time_to_live(live_servicebus_config, standard_queue): client = ServiceBusClient( service_namespace=live_servicebus_config['hostname'], shared_access_key_name=live_servicebus_config['key_name'], shared_access_key_value=live_servicebus_config['access_key'], debug=True) import uuid queue_client = client.get_queue(standard_queue) with queue_client.get_sender() as sender: content = str(uuid.uuid4()) message_id = uuid.uuid4() message = Message(content) message.time_to_live = timedelta(seconds=30) sender.send(message) time.sleep(30) with queue_client.get_receiver() as receiver: messages = receiver.fetch_next(timeout=10) assert not messages with queue_client.get_deadletter_receiver( idle_timeout=5, mode=ReceiveSettleMode.PeekLock) as receiver: count = 0 for message in receiver: print_message(message) message.complete() count += 1 assert count == 1
def test_session_schedule_message(live_servicebus_config, session_queue): client = ServiceBusClient( service_namespace=live_servicebus_config['hostname'], shared_access_key_name=live_servicebus_config['key_name'], shared_access_key_value=live_servicebus_config['access_key'], debug=True) import uuid session_id = str(uuid.uuid4()) queue_client = client.get_queue(session_queue) enqueue_time = (datetime.now() + timedelta(minutes=2)).replace(microsecond=0) with queue_client.get_receiver(session=session_id) as receiver: with queue_client.get_sender(session=session_id) as sender: content = str(uuid.uuid4()) message_id = uuid.uuid4() message = Message(content) message.properties.message_id = message_id message.schedule(enqueue_time) sender.send(message) messages = [] count = 0 while not messages and count < 12: messages = receiver.fetch_next(timeout=10) receiver.renew_lock() count += 1 data = str(messages[0]) assert data == content assert messages[0].properties.message_id == message_id assert messages[0].scheduled_enqueue_time == enqueue_time assert messages[0].scheduled_enqueue_time == messages[0].enqueued_time.replace(microsecond=0) assert len(messages) == 1
def sample_session_send_receive_with_pool(sb_config, queue): concurrent_receivers = 5 sessions = [str(uuid.uuid4()) for i in range(2 * concurrent_receivers)] client = ServiceBusClient( service_namespace=sb_config['hostname'], shared_access_key_name=sb_config['key_name'], shared_access_key_value=sb_config['access_key'], debug=True) queue_client = client.get_queue(queue) for session_id in sessions: with queue_client.get_sender(session=session_id) as sender: for i in range(20): message = Message("Sample message no. {}".format(i)) sender.send(message) all_messages = [] futures = [] with concurrent.futures.ThreadPoolExecutor(max_workers=concurrent_receivers) as thread_pool: for _ in range(concurrent_receivers): futures.append(thread_pool.submit(message_processing, queue_client, all_messages)) concurrent.futures.wait(futures) print("Received total {} messages across {} sessions.".format(len(all_messages), 2*concurrent_receivers))
def test_session_by_servicebus_client_list_sessions_with_receiver(live_servicebus_config, session_queue): client = ServiceBusClient( service_namespace=live_servicebus_config['hostname'], shared_access_key_name=live_servicebus_config['key_name'], shared_access_key_value=live_servicebus_config['access_key'], debug=True) queue_client = client.get_queue(session_queue) sessions = [] start_time = datetime.now() for i in range(5): sessions.append(str(uuid.uuid4())) for session in sessions: with queue_client.get_sender(session=session) as sender: for i in range(5): message = Message("Test message no. {}".format(i)) sender.send(message) for session in sessions: with queue_client.get_receiver(session=session) as receiver: receiver.set_session_state("SESSION {}".format(session)) with queue_client.get_receiver(session=NEXT_AVAILABLE, idle_timeout=5, mode=ReceiveSettleMode.PeekLock) as receiver: current_sessions = receiver.list_sessions(updated_since=start_time) assert len(current_sessions) == 5 assert current_sessions == sessions
def test_queue_by_servicebus_client_renew_message_locks(live_servicebus_config, standard_queue): client = ServiceBusClient( service_namespace=live_servicebus_config['hostname'], shared_access_key_name=live_servicebus_config['key_name'], shared_access_key_value=live_servicebus_config['access_key'], debug=True) queue_client = client.get_queue(standard_queue) messages = [] locks = 3 with queue_client.get_receiver(idle_timeout=5, mode=ReceiveSettleMode.PeekLock, prefetch=10) as receiver: with queue_client.get_sender() as sender: for i in range(locks): message = Message("Test message no. {}".format(i)) sender.send(message) messages.extend(receiver.fetch_next()) recv = True while recv: recv = receiver.fetch_next() messages.extend(recv) try: assert not message.expired for m in messages: time.sleep(5) initial_expiry = m.locked_until m.renew_lock() assert (m.locked_until - initial_expiry) >= timedelta(seconds=5) finally: messages[0].complete() messages[1].complete() time.sleep(30) with pytest.raises(MessageLockExpired): messages[2].complete()
def test_queue_message_duplicate_detection(live_servicebus_config, duplicate_queue): client = ServiceBusClient( service_namespace=live_servicebus_config['hostname'], shared_access_key_name=live_servicebus_config['key_name'], shared_access_key_value=live_servicebus_config['access_key'], debug=True) import uuid message_id = uuid.uuid4() queue_client = client.get_queue(duplicate_queue) with queue_client.get_sender() as sender: for i in range(5): message = Message(str(i)) message.properties.message_id = message_id sender.send(message) with queue_client.get_receiver(idle_timeout=5) as receiver: count = 0 for message in receiver: print_message(message) assert message.properties.message_id == message_id message.complete() count += 1 assert count == 1
def test_session_by_servicebus_client_list_sessions_with_client( self, servicebus_namespace, servicebus_namespace_key_name, servicebus_namespace_primary_key, servicebus_queue, **kwargs): client = ServiceBusClient( service_namespace=servicebus_namespace.name, shared_access_key_name=servicebus_namespace_key_name, shared_access_key_value=servicebus_namespace_primary_key, debug=False) queue_client = client.get_queue(servicebus_queue.name) sessions = [] start_time = datetime.now() for i in range(5): sessions.append(str(uuid.uuid4())) for session in sessions: with queue_client.get_sender(session=session) as sender: for i in range(5): message = Message("Test message no. {}".format(i)) sender.send(message) for session in sessions: with queue_client.get_receiver(session=session) as receiver: receiver.set_session_state("SESSION {}".format(session)) current_sessions = queue_client.list_sessions(updated_since=start_time) assert len(current_sessions) == 5 assert current_sessions == sessions
def test_queue_by_servicebus_client_iter_messages_with_retrieve_deferred_receiver_complete(live_servicebus_config, standard_queue): client = ServiceBusClient( service_namespace=live_servicebus_config['hostname'], shared_access_key_name=live_servicebus_config['key_name'], shared_access_key_value=live_servicebus_config['access_key'], debug=True) queue_client = client.get_queue(standard_queue) deferred_messages = [] messages = [Message("Deferred message no. {}".format(i)) for i in range(10)] results = queue_client.send(messages, session="test_session") assert all(result[0] for result in results) with queue_client.get_receiver(idle_timeout=5, mode=ReceiveSettleMode.PeekLock) as receiver: count = 0 for message in receiver: deferred_messages.append(message.sequence_number) print_message(message) count += 1 message.defer() assert count == 10 with queue_client.get_receiver(idle_timeout=5, mode=ReceiveSettleMode.PeekLock) as receiver: deferred = receiver.receive_deferred_messages(deferred_messages) assert len(deferred) == 10 for message in deferred: assert isinstance(message, DeferredMessage) assert message.lock_token assert message.locked_until assert message._receiver message.renew_lock() message.complete()
def test_queue_message_expiry(live_servicebus_config, standard_queue): client = ServiceBusClient( service_namespace=live_servicebus_config['hostname'], shared_access_key_name=live_servicebus_config['key_name'], shared_access_key_value=live_servicebus_config['access_key'], debug=True) import uuid queue_client = client.get_queue(standard_queue) with queue_client.get_sender() as sender: content = str(uuid.uuid4()) message = Message(content) sender.send(message) with queue_client.get_receiver() as receiver: messages = receiver.fetch_next(timeout=10) assert len(messages) == 1 time.sleep(30) assert messages[0].expired with pytest.raises(MessageLockExpired): messages[0].complete() with pytest.raises(MessageLockExpired): messages[0].renew_lock() with queue_client.get_receiver() as receiver: messages = receiver.fetch_next(timeout=30) assert len(messages) == 1 print_message(messages[0]) assert messages[0].header.delivery_count > 0 messages[0].complete()
def test_pqueue_cancel_scheduled_messages(live_servicebus_config, partitioned_queue): client = ServiceBusClient( service_namespace=live_servicebus_config['hostname'], shared_access_key_name=live_servicebus_config['key_name'], shared_access_key_value=live_servicebus_config['access_key'], debug=True) queue_client = client.get_queue(partitioned_queue) enqueue_time = (datetime.now() + timedelta(minutes=2)).replace(microsecond=0) with queue_client.get_receiver() as receiver: with queue_client.get_sender() as sender: message_a = Message("Test scheduled message") message_b = Message("Test scheduled message") tokens = sender.schedule(enqueue_time, message_a, message_b) assert len(tokens) == 2 sender.cancel_scheduled_messages(*tokens) messages = receiver.fetch_next(timeout=120) try: assert len(messages) == 0 except AssertionError: for m in messages: print(str(m)) m.complete() raise
def sample_queue_send_receive_batch(sb_config, queue): client = ServiceBusClient( service_namespace=sb_config['hostname'], shared_access_key_name=sb_config['key_name'], shared_access_key_value=sb_config['access_key'], debug=True) queue_client = client.get_queue(queue) with queue_client.get_sender() as sender: for i in range(100): message = Message("Sample message no. {}".format(i)) sender.send(message) with queue_client.get_receiver(idle_timeout=1, mode=ReceiveSettleMode.PeekLock, prefetch=10) as receiver: total = 0 # Receive list of messages as a batch batch = receiver.fetch_next(max_batch_size=10) for message in batch: print("Message: {}".format(message)) print("Sequence number: {}".format(message.sequence_number)) message.complete() total += 1 # Receive messages as a continuous generator for message in receiver: print("Message: {}".format(message)) print("Sequence number: {}".format(message.sequence_number)) message.complete() total += 1 print("Received total {} messages".format(total))
def test_pqueue_by_servicebus_client_iter_messages_simple(live_servicebus_config, partitioned_queue): client = ServiceBusClient( service_namespace=live_servicebus_config['hostname'], shared_access_key_name=live_servicebus_config['key_name'], shared_access_key_value=live_servicebus_config['access_key'], debug=True) queue_client = client.get_queue(partitioned_queue) with queue_client.get_receiver(idle_timeout=5, mode=ReceiveSettleMode.PeekLock) as receiver: with queue_client.get_sender() as sender: for i in range(10): message = Message("Iter message no. {}".format(i)) sender.send(message) count = 0 for message in receiver: print_message(message) message.complete() with pytest.raises(MessageAlreadySettled): message.complete() with pytest.raises(MessageAlreadySettled): message.renew_lock() count += 1 with pytest.raises(InvalidHandlerState): next(receiver) assert count == 10
def test_pqueue_by_servicebus_client_fail_send_batch_messages(live_servicebus_config, partitioned_queue): pytest.skip("TODO: Pending bugfix in uAMQP") def batch_data(): for i in range(3): yield str(i) * 1024 * 256 client = ServiceBusClient( service_namespace=live_servicebus_config['hostname'], shared_access_key_name=live_servicebus_config['key_name'], shared_access_key_value=live_servicebus_config['access_key'], debug=True) queue_client = client.get_queue(partitioned_queue) results = queue_client.send(BatchMessage(batch_data())) assert len(results) == 4 assert not results[0][0] assert isinstance(results[0][1], MessageSendFailed) with queue_client.get_sender() as sender: with pytest.raises(MessageSendFailed): sender.send(BatchMessage(batch_data())) with queue_client.get_sender() as sender: sender.queue_message(BatchMessage(batch_data())) results = sender.send_pending_messages() assert len(results) == 4 assert not results[0][0] assert isinstance(results[0][1], MessageSendFailed)
def test_pqueue_message_expiry(live_servicebus_config, partitioned_queue): client = ServiceBusClient( service_namespace=live_servicebus_config['hostname'], shared_access_key_name=live_servicebus_config['key_name'], shared_access_key_value=live_servicebus_config['access_key'], debug=True) import uuid queue_client = client.get_queue(partitioned_queue) with queue_client.get_sender() as sender: content = str(uuid.uuid4()) message = Message(content) sender.send(message) with queue_client.get_receiver() as receiver: messages = receiver.fetch_next(timeout=10) assert len(messages) == 1 time.sleep(30) assert messages[0].expired with pytest.raises(MessageLockExpired): messages[0].complete() with pytest.raises(MessageLockExpired): messages[0].renew_lock() with queue_client.get_receiver() as receiver: messages = receiver.fetch_next(timeout=30) assert len(messages) == 1 print_message(messages[0]) assert messages[0].header.delivery_count > 0 messages[0].complete()
def test_pqueue_message_time_to_live(live_servicebus_config, partitioned_queue): client = ServiceBusClient( service_namespace=live_servicebus_config['hostname'], shared_access_key_name=live_servicebus_config['key_name'], shared_access_key_value=live_servicebus_config['access_key'], debug=True) import uuid queue_client = client.get_queue(partitioned_queue) with queue_client.get_sender() as sender: content = str(uuid.uuid4()) message_id = uuid.uuid4() message = Message(content) message.time_to_live = timedelta(seconds=30) sender.send(message) time.sleep(30) with queue_client.get_receiver() as receiver: messages = receiver.fetch_next(timeout=10) assert not messages with queue_client.get_deadletter_receiver(idle_timeout=5, mode=ReceiveSettleMode.PeekLock) as receiver: count = 0 for message in receiver: print_message(message) message.complete() count += 1 assert count == 1
def test_pqueue_message_receive_and_delete(live_servicebus_config, partitioned_queue): client = ServiceBusClient( service_namespace=live_servicebus_config['hostname'], shared_access_key_name=live_servicebus_config['key_name'], shared_access_key_value=live_servicebus_config['access_key'], debug=True) queue_client = client.get_queue(partitioned_queue) with queue_client.get_sender() as sender: message = Message("Receive and delete test") sender.send(message) with queue_client.get_receiver(mode=ReceiveSettleMode.ReceiveAndDelete) as receiver: messages = receiver.fetch_next(timeout=10) assert len(messages) == 1 received = messages[0] print_message(received) with pytest.raises(MessageAlreadySettled): received.complete() with pytest.raises(MessageAlreadySettled): received.abandon() with pytest.raises(MessageAlreadySettled): received.defer() with pytest.raises(MessageAlreadySettled): received.dead_letter() with pytest.raises(MessageAlreadySettled): received.renew_lock() time.sleep(30) with queue_client.get_receiver() as receiver: messages = receiver.fetch_next(timeout=10) for m in messages: print_message(m) assert len(messages) == 0
def test_pqueue_by_servicebus_client_iter_messages_with_retrieve_deferred_not_found(live_servicebus_config, partitioned_queue): client = ServiceBusClient( service_namespace=live_servicebus_config['hostname'], shared_access_key_name=live_servicebus_config['key_name'], shared_access_key_value=live_servicebus_config['access_key'], debug=True) queue_client = client.get_queue(partitioned_queue) deferred_messages = [] with queue_client.get_receiver(idle_timeout=5, mode=ReceiveSettleMode.PeekLock) as receiver: with queue_client.get_sender() as sender: for i in range(3): message = Message("Deferred message no. {}".format(i)) message.partition_key = "MyPartitionKey" sender.send(message) count = 0 for message in receiver: deferred_messages.append(message.sequence_number) print_message(message) count += 1 message.defer() assert count == 3 with pytest.raises(ServiceBusError): deferred = queue_client.receive_deferred_messages([3, 4], mode=ReceiveSettleMode.PeekLock) with pytest.raises(ServiceBusError): deferred = queue_client.receive_deferred_messages([5, 6, 7], mode=ReceiveSettleMode.PeekLock)
def test_session_by_servicebus_client_iter_messages_with_retrieve_deferred_receiver_complete(live_servicebus_config, session_queue): client = ServiceBusClient( service_namespace=live_servicebus_config['hostname'], shared_access_key_name=live_servicebus_config['key_name'], shared_access_key_value=live_servicebus_config['access_key'], debug=True) queue_client = client.get_queue(session_queue) deferred_messages = [] session_id = str(uuid.uuid4()) messages = [Message("Deferred message no. {}".format(i)) for i in range(10)] results = queue_client.send(messages, session=session_id) assert all(result[0] for result in results) count = 0 session = queue_client.get_receiver(session=session_id, idle_timeout=5) for message in session: deferred_messages.append(message.sequence_number) print_message(message) count += 1 message.defer() assert count == 10 with queue_client.get_receiver(session=session_id, idle_timeout=5) as session: deferred = session.receive_deferred_messages(deferred_messages) assert len(deferred) == 10 for message in deferred: assert isinstance(message, DeferredMessage) assert message.lock_token assert not message.locked_until assert message._receiver with pytest.raises(TypeError): message.renew_lock() message.complete()
def test_pqueue_message_batch(live_servicebus_config, partitioned_queue): client = ServiceBusClient( service_namespace=live_servicebus_config['hostname'], shared_access_key_name=live_servicebus_config['key_name'], shared_access_key_value=live_servicebus_config['access_key'], debug=True) queue_client = client.get_queue(partitioned_queue) def message_content(): for i in range(5): yield "Message no. {}".format(i) with queue_client.get_sender() as sender: message = BatchMessage(message_content()) sender.send(message) with queue_client.get_receiver() as receiver: messages =receiver.fetch_next(timeout=10) recv = True while recv: recv = receiver.fetch_next(timeout=10) messages.extend(recv) assert len(messages) == 5 for m in messages: print_message(m) m.complete()
def test_pqueue_schedule_message(live_servicebus_config, partitioned_queue): client = ServiceBusClient( service_namespace=live_servicebus_config['hostname'], shared_access_key_name=live_servicebus_config['key_name'], shared_access_key_value=live_servicebus_config['access_key'], debug=True) import uuid queue_client = client.get_queue(partitioned_queue) enqueue_time = (datetime.now() + timedelta(minutes=2)).replace(microsecond=0) with queue_client.get_receiver() as receiver: with queue_client.get_sender() as sender: content = str(uuid.uuid4()) message_id = uuid.uuid4() message = Message(content) message.properties.message_id = message_id message.schedule(enqueue_time) sender.send(message) messages = receiver.fetch_next(timeout=120) if messages: try: data = str(messages[0]) assert data == content assert messages[0].properties.message_id == message_id assert messages[0].scheduled_enqueue_time == enqueue_time assert messages[0].scheduled_enqueue_time == messages[0].enqueued_time.replace(microsecond=0) assert len(messages) == 1 finally: for m in messages: m.complete() else: raise Exception("Failed to receive schdeduled message.")
def test_session_by_servicebus_client_browse_messages_client( self, servicebus_namespace, servicebus_namespace_key_name, servicebus_namespace_primary_key, servicebus_queue, **kwargs): client = ServiceBusClient( service_namespace=servicebus_namespace.name, shared_access_key_name=servicebus_namespace_key_name, shared_access_key_value=servicebus_namespace_primary_key, debug=False) queue_client = client.get_queue(servicebus_queue.name) session_id = str(uuid.uuid4()) with queue_client.get_sender(session=session_id) as sender: for i in range(5): message = Message("Test message no. {}".format(i)) sender.send(message) with pytest.raises(ValueError): messages = queue_client.peek(5) messages = queue_client.peek(5, session=session_id) assert len(messages) == 5 assert all(isinstance(m, PeekMessage) for m in messages) for message in messages: print_message(message) with pytest.raises(TypeError): message.complete()
def test_pqueue_by_servicebus_client_iter_messages_with_retrieve_deferred_receiver_deletemode(live_servicebus_config, partitioned_queue): client = ServiceBusClient( service_namespace=live_servicebus_config['hostname'], shared_access_key_name=live_servicebus_config['key_name'], shared_access_key_value=live_servicebus_config['access_key'], debug=True) queue_client = client.get_queue(partitioned_queue) deferred_messages = [] messages = [Message("Deferred message no. {}".format(i)) for i in range(10)] for m in messages: m.partition_key = "MyPartitionKey" results = queue_client.send(messages) assert all(result[0] for result in results) count = 0 receiver = queue_client.get_receiver(idle_timeout=5) for message in receiver: deferred_messages.append(message.sequence_number) print_message(message) count += 1 message.defer() assert count == 10 with queue_client.get_receiver(idle_timeout=5) as receiver: deferred = receiver.receive_deferred_messages(deferred_messages, mode=ReceiveSettleMode.ReceiveAndDelete) assert len(deferred) == 10 for message in deferred: assert isinstance(message, DeferredMessage) with pytest.raises(MessageAlreadySettled): message.complete() with pytest.raises(ServiceBusError): deferred = receiver.receive_deferred_messages(deferred_messages)
def test_session_by_servicebus_client_list_sessions_with_client(live_servicebus_config, session_queue): client = ServiceBusClient( service_namespace=live_servicebus_config['hostname'], shared_access_key_name=live_servicebus_config['key_name'], shared_access_key_value=live_servicebus_config['access_key'], debug=True) queue_client = client.get_queue(session_queue) sessions = [] start_time = datetime.now() for i in range(5): sessions.append(str(uuid.uuid4())) for session in sessions: with queue_client.get_sender(session=session) as sender: for i in range(5): message = Message("Test message no. {}".format(i)) sender.send(message) for session in sessions: with queue_client.get_receiver(session=session) as receiver: receiver.set_session_state("SESSION {}".format(session)) current_sessions = queue_client.list_sessions(updated_since=start_time) assert len(current_sessions) == 5 assert current_sessions == sessions
def test_pqueue_by_servicebus_client_receive_batch_with_deadletter(live_servicebus_config, partitioned_queue): client = ServiceBusClient( service_namespace=live_servicebus_config['hostname'], shared_access_key_name=live_servicebus_config['key_name'], shared_access_key_value=live_servicebus_config['access_key'], debug=True) queue_client = client.get_queue(partitioned_queue) with queue_client.get_receiver(idle_timeout=5, mode=ReceiveSettleMode.PeekLock, prefetch=10) as receiver: with queue_client.get_sender() as sender: for i in range(10): message = Message("Dead lettered message no. {}".format(i)) sender.send(message) count = 0 messages = receiver.fetch_next() while messages: for message in messages: print_message(message) count += 1 message.dead_letter(description="Testing") messages = receiver.fetch_next() assert count == 10 with queue_client.get_receiver(idle_timeout=5, mode=ReceiveSettleMode.PeekLock) as receiver: count = 0 for message in receiver: print_message(message) message.complete() count += 1 assert count == 0
def test_pqueue_message_lock_renew(live_servicebus_config, partitioned_queue): client = ServiceBusClient( service_namespace=live_servicebus_config['hostname'], shared_access_key_name=live_servicebus_config['key_name'], shared_access_key_value=live_servicebus_config['access_key'], debug=True) import uuid queue_client = client.get_queue(partitioned_queue) with queue_client.get_sender() as sender: content = str(uuid.uuid4()) message = Message(content) sender.send(message) with queue_client.get_receiver() as receiver: messages = receiver.fetch_next(timeout=10) assert len(messages) == 1 time.sleep(15) messages[0].renew_lock() time.sleep(15) messages[0].renew_lock() time.sleep(15) assert not messages[0].expired messages[0].complete() with queue_client.get_receiver() as receiver: messages = receiver.fetch_next(timeout=10) assert len(messages) == 0
def test_pqueue_by_servicebus_client_fail_send_messages(live_servicebus_config, partitioned_queue): client = ServiceBusClient( service_namespace=live_servicebus_config['hostname'], shared_access_key_name=live_servicebus_config['key_name'], shared_access_key_value=live_servicebus_config['access_key'], debug=True) queue_client = client.get_queue(partitioned_queue) too_large = "A" * 1024 * 512 try: results = queue_client.send(Message(too_large)) except MessageSendFailed: pytest.skip("Open issue for uAMQP on OSX") assert len(results) == 1 assert not results[0][0] assert isinstance(results[0][1], MessageSendFailed) with queue_client.get_sender() as sender: with pytest.raises(MessageSendFailed): sender.send(Message(too_large)) with queue_client.get_sender() as sender: sender.queue_message(Message(too_large)) results = sender.send_pending_messages() assert len(results) == 1 assert not results[0][0] assert isinstance(results[0][1], MessageSendFailed)
def test_pqueue_by_servicebus_client_iter_messages_with_defer(live_servicebus_config, partitioned_queue): client = ServiceBusClient( service_namespace=live_servicebus_config['hostname'], shared_access_key_name=live_servicebus_config['key_name'], shared_access_key_value=live_servicebus_config['access_key'], debug=True) queue_client = client.get_queue(partitioned_queue) deferred_messages = [] with queue_client.get_receiver(idle_timeout=5, mode=ReceiveSettleMode.PeekLock) as receiver: with queue_client.get_sender() as sender: for i in range(10): message = Message("Deferred message no. {}".format(i)) sender.send(message) count = 0 for message in receiver: deferred_messages.append(message.sequence_number) print_message(message) count += 1 message.defer() assert count == 10 with queue_client.get_receiver(idle_timeout=5, mode=ReceiveSettleMode.PeekLock) as receiver: count = 0 for message in receiver: print_message(message) message.complete() count += 1 assert count == 0
def test_pqueue_by_servicebus_client_browse_empty_messages(live_servicebus_config, partitioned_queue): client = ServiceBusClient( service_namespace=live_servicebus_config['hostname'], shared_access_key_name=live_servicebus_config['key_name'], shared_access_key_value=live_servicebus_config['access_key'], debug=True) queue_client = client.get_queue(partitioned_queue) with queue_client.get_receiver(idle_timeout=5, mode=ReceiveSettleMode.PeekLock, prefetch=10) as receiver: messages = receiver.peek(10) assert len(messages) == 0
def resend_dead_letters_from_queue(vault_dir: str, queue: str, client: ServiceBusClient) -> List[str]: """Scan a queue for dead letters, and resend found. If there are any errors resending, store the message body in the vault. Args: vault_dir (str): The root dir of the vault. queue (str): The name of the queue to scan. client (ServiceBusClient): The service bus client to use. Returns: List[str]: The list of transaction IDs that were resent. """ logging.info(f"--> Getting dead letters from '{queue}'...") resent_ids = [] queue = client.get_queue(queue) with queue.get_deadletter_receiver(idle_timeout=0.2) as dead_letter_rx: with queue.get_sender() as queue_sender: number_replayed = 0 # receive the messages from the dead letter queue for msg in dead_letter_rx: message_id = msg.properties.message_id.decode('utf-8') # messages without a body are invalid, we should skip those if msg.body is None: logging.warn( f"\tMessage '{message_id}' did not have a body.") msg.complete() continue try: # now attempt to replay the error logging.info( f"\tReplaying a dead letter ({message_id})...") store_message_in_vault(msg, vault_dir) to_send = copy_message(msg) queue_sender.send(to_send) number_replayed += 1 resent_ids.append(message_id) except: # if we get any errors while replaying, make sure the message # is abandoned, and kept on the dead letter queue logging.error("--> An error occurred while processing a dead letter."\ " Abandoning the message.") msg.abandon() return msg.complete() logging.info( f" Finished scanning. {number_replayed} dead letter(s) were found and replayed." ) return resent_ids
def message_send_process(sb_config, queue, endtime): client = ServiceBusClient(service_namespace=sb_config['hostname'], shared_access_key_name=sb_config['key_name'], shared_access_key_value=sb_config['access_key'], debug=True) total = 0 queue_client = client.get_queue(queue) while endtime > datetime.now(): queue_client.send(Message("Slow stress test message")) total += 1 time.sleep(3) return total
def test_session_by_servicebus_client_renew_client_locks( self, servicebus_namespace, servicebus_namespace_key_name, servicebus_namespace_primary_key, servicebus_queue, **kwargs): client = ServiceBusClient( service_namespace=servicebus_namespace.name, shared_access_key_name=servicebus_namespace_key_name, shared_access_key_value=servicebus_namespace_primary_key, debug=False) queue_client = client.get_queue(servicebus_queue.name) session_id = str(uuid.uuid4()) messages = [] locks = 3 with queue_client.get_receiver(session=session_id, prefetch=10) as receiver: with queue_client.get_sender(session=session_id) as sender: for i in range(locks): message = Message("Test message no. {}".format(i)) sender.send(message) messages.extend(receiver.fetch_next()) recv = True while recv: recv = receiver.fetch_next(timeout=5) messages.extend(recv) try: for m in messages: with pytest.raises(TypeError): expired = m.expired assert m.locked_until is None assert m.lock_token is None time.sleep(5) initial_expiry = receiver.locked_until receiver.renew_lock() assert (receiver.locked_until - initial_expiry) >= timedelta(seconds=5) finally: messages[0].complete() messages[1].complete() # This magic number is because of a 30 second lock renewal window. Chose 31 seconds because at 30, you'll see "off by .05 seconds" flaky failures # potentially as a side effect of network delays/sleeps/"typical distributed systems nonsense." In a perfect world we wouldn't have a magic number/network hop but this allows # a slightly more robust test in absence of that. assert (receiver.locked_until - datetime.now()) <= timedelta(seconds=31) time.sleep( (receiver.locked_until - datetime.now()).total_seconds()) with pytest.raises(SessionLockExpired): messages[2].complete()
def test_pqueue_by_servicebus_client_session_fail(live_servicebus_config, partitioned_queue): client = ServiceBusClient( service_namespace=live_servicebus_config['hostname'], shared_access_key_name=live_servicebus_config['key_name'], shared_access_key_value=live_servicebus_config['access_key'], debug=True) queue_client = client.get_queue(partitioned_queue) with pytest.raises(ValueError): queue_client.get_receiver(session="test") with queue_client.get_sender(session="test") as sender: sender.send(Message("test session sender"))
def queue_sender(sb_config, queue): client = ServiceBusClient( service_namespace=sb_config['hostname'], shared_access_key_name=sb_config['key_name'], shared_access_key_value=sb_config['access_key'], debug=False) queue_client = client.get_queue(queue) with queue_client.get_sender() as sender: while True: for i in range(100): message = Message("Publishing from {}: Sample message no. {}".format(live_config['pub_name'],i)) sender.send(message) sleep(randrange(9))
def queue_rcv(sb_config, queue): client = ServiceBusClient(service_namespace=sb_config['hostname'], shared_access_key_name=sb_config['key_name'], shared_access_key_value=sb_config['access_key'], debug=False) queue_client = client.get_queue(queue) with queue_client.get_receiver(mode=ReceiveSettleMode.PeekLock, prefetch=10) as receiver: # Receive messages as a continuous generator for message in receiver: print("Message: {}".format(message)) print("Sequence number: {}".format(message.sequence_number)) message.complete()
def test_session_by_servicebus_client_session_pool( self, servicebus_namespace, servicebus_namespace_key_name, servicebus_namespace_primary_key, servicebus_queue, **kwargs): messages = [] errors = [] concurrent_receivers = 5 def message_processing(queue_client): while True: try: with queue_client.get_receiver(session=NEXT_AVAILABLE, idle_timeout=5) as session: for message in session: print("Message: {}".format(message)) messages.append(message) message.complete() except NoActiveSession: return except Exception as e: errors.append(e) raise client = ServiceBusClient( service_namespace=servicebus_namespace.name, shared_access_key_name=servicebus_namespace_key_name, shared_access_key_value=servicebus_namespace_primary_key, debug=False) queue_client = client.get_queue(servicebus_queue.name) sessions = [str(uuid.uuid4()) for i in range(concurrent_receivers)] for session in sessions: with queue_client.get_sender(session=session) as sender: for i in range(20): message = Message("Test message no. {}".format(i)) sender.send(message) futures = [] with concurrent.futures.ThreadPoolExecutor( max_workers=concurrent_receivers) as thread_pool: for _ in range(concurrent_receivers): futures.append( thread_pool.submit(message_processing, queue_client)) concurrent.futures.wait(futures) assert not errors assert len(messages) == 100
def message_receive_process(sb_config, queue, endtime): client = ServiceBusClient(service_namespace=sb_config['hostname'], shared_access_key_name=sb_config['key_name'], shared_access_key_value=sb_config['access_key'], debug=True) queue_client = client.get_queue(queue) total = 0 while endtime > datetime.now(): with queue_client.get_receiver() as receiver: batch = receiver.fetch_next() for message in batch: total += 1 message.complete() return total