def handle_receive_delivered(
chain_state: ChainState, state_change: ReceiveDelivered
) -> TransitionResult[ChainState]:
""" Check if the "Delivered" message exists in the global queue and delete if found."""
queueid = QueueIdentifier(state_change.sender, CANONICAL_IDENTIFIER_UNORDERED_QUEUE)
inplace_delete_message_queue(chain_state, state_change, queueid)
return TransitionResult(chain_state, [])
def test_channel_closed_must_clear_ordered_messages(chain_state,
token_network_state,
netting_channel_state):
recipient = netting_channel_state.partner_state.address
message_identifier = random.randint(0, 2**16)
amount = 10
queue_identifier = QueueIdentifier(
recipient=recipient,
canonical_identifier=netting_channel_state.canonical_identifier)
# Regression test:
# The code delivered_message handler worked only with a queue of one
# element
message = factories.create(
factories.LockedTransferProperties(
message_identifier=message_identifier,
token=token_network_state.token_address,
canonical_identifier=netting_channel_state.canonical_identifier,
transferred_amount=amount,
recipient=recipient,
))
chain_state.queueids_to_queues[queue_identifier] = [message]
closed = state_change.ContractReceiveChannelClosed(
transaction_hash=EMPTY_HASH,
transaction_from=recipient,
canonical_identifier=netting_channel_state.canonical_identifier,
block_number=1,
block_hash=factories.make_block_hash(),
)
iteration = node.handle_state_change(chain_state, closed)
assert queue_identifier not in iteration.new_state.queueids_to_queues
def test_restore_queueids_to_queues(chain_state, netting_channel_state):
"""Test that withdraw messages are restorable if they exist in
chain_state.queueids_to_queues.
"""
recipient = netting_channel_state.partner_state.address
queue_identifier = QueueIdentifier(
recipient=recipient,
canonical_identifier=netting_channel_state.canonical_identifier)
msg_args = dict(
recipient=recipient,
canonical_identifier=netting_channel_state.canonical_identifier,
message_identifier=factories.make_message_identifier(),
total_withdraw=WithdrawAmount(1),
participant=recipient,
expiration=BlockExpiration(10),
nonce=Nonce(15),
)
messages = [
SendWithdrawRequest(**msg_args),
SendWithdrawConfirmation(**msg_args),
SendWithdrawExpired(**msg_args),
]
chain_state.queueids_to_queues[queue_identifier] = messages
serialized_chain_state = JSONSerializer.serialize(chain_state)
deserialized_chain_state = JSONSerializer.deserialize(
serialized_chain_state)
assert chain_state == deserialized_chain_state
def test_retryqueue_not_idle_with_messages(
mock_matrix: MatrixTransport, retry_interval_initial: float
) -> None:
""" Ensure ``RetryQueue``s don't become idle while messages remain in the internal queue. """
retry_queue = mock_matrix._get_retrier(Address(factories.HOP1))
idle_after = RETRY_QUEUE_IDLE_AFTER * retry_interval_initial
queue_identifier = QueueIdentifier(
recipient=Address(factories.HOP1),
canonical_identifier=CANONICAL_IDENTIFIER_UNORDERED_QUEUE,
)
retry_queue.enqueue(queue_identifier, [make_message()])
# Without the `all_peers_reachable` fixture, the default reachability will be `UNREACHABLE`
# therefore the message will remain in the internal queue indefinitely.
# Wait for the idle timeout to expire
gevent.sleep(idle_after + (retry_interval_initial * 5))
assert not retry_queue.greenlet.ready()
assert retry_queue._idle_since == 0
assert not retry_queue.is_idle
retry_queue_2 = mock_matrix._get_retrier(Address(factories.HOP1))
# The first queue has never become idle, therefore the same object must be returned
assert retry_queue is retry_queue_2
def ping_pong_message_success(transport0, transport1):
queueid0 = QueueIdentifier(
recipient=transport0._raiden_service.address,
channel_identifier=CHANNEL_IDENTIFIER_GLOBAL_QUEUE,
)
queueid1 = QueueIdentifier(
recipient=transport1._raiden_service.address,
channel_identifier=CHANNEL_IDENTIFIER_GLOBAL_QUEUE,
)
received_messages0 = transport0._raiden_service.message_handler.bag
received_messages1 = transport1._raiden_service.message_handler.bag
msg_id = random.randint(1e5, 9e5)
ping_message = Processed(message_identifier=msg_id)
pong_message = Delivered(delivered_message_identifier=msg_id)
transport0._raiden_service.sign(ping_message)
transport1._raiden_service.sign(pong_message)
transport0.send_async(queueid1, ping_message)
with Timeout(20, exception=False):
all_messages_received = False
while not all_messages_received:
all_messages_received = (ping_message in received_messages1
and pong_message in received_messages0)
gevent.sleep(0.1)
assert ping_message in received_messages1
assert pong_message in received_messages0
transport0._raiden_service.sign(pong_message)
transport1._raiden_service.sign(ping_message)
transport1.send_async(queueid0, ping_message)
with Timeout(20, exception=False):
all_messages_received = False
while not all_messages_received:
all_messages_received = (ping_message in received_messages0
and pong_message in received_messages1)
gevent.sleep(0.1)
assert ping_message in received_messages0
assert pong_message in received_messages1
return all_messages_received
def test_inplace_delete_message_queue(chain_state):
sender = factories.make_address()
canonical_identifier = factories.make_canonical_identifier()
message_id = factories.make_message_identifier()
delivered_state_change = ReceiveDelivered(sender=sender,
message_identifier=message_id)
processed_state_change = ReceiveProcessed(sender=sender,
message_identifier=message_id)
global_identifier = QueueIdentifier(
recipient=sender,
canonical_identifier=CANONICAL_IDENTIFIER_GLOBAL_QUEUE)
chain_state.queueids_to_queues[global_identifier] = None
assert global_identifier in chain_state.queueids_to_queues, "queue mapping insertion failed"
inplace_delete_message_queue(chain_state=chain_state,
state_change=delivered_state_change,
queueid=global_identifier)
assert global_identifier not in chain_state.queueids_to_queues, "did not clear queue"
chain_state.queueids_to_queues[global_identifier] = [
SendMessageEvent(
recipient=sender,
canonical_identifier=canonical_identifier,
message_identifier=message_id,
)
]
assert global_identifier in chain_state.queueids_to_queues, "queue mapping insertion failed"
handle_receive_delivered(chain_state=chain_state,
state_change=delivered_state_change)
assert global_identifier not in chain_state.queueids_to_queues, "did not clear queue"
queue_identifier = QueueIdentifier(
recipient=sender, canonical_identifier=canonical_identifier)
assert queue_identifier not in chain_state.queueids_to_queues, "queue not empty"
chain_state.queueids_to_queues[queue_identifier] = [
SendMessageEvent(
recipient=sender,
canonical_identifier=canonical_identifier,
message_identifier=message_id,
)
]
assert queue_identifier in chain_state.queueids_to_queues, "queue mapping not mutable"
handle_receive_processed(chain_state=chain_state,
state_change=processed_state_change)
assert queue_identifier not in chain_state.queueids_to_queues, "queue did not clear"
def _deserialize(self, value: str, attr: Any, data: Any,
**kwargs: Any) -> QueueIdentifier:
try:
str_recipient, str_canonical_id = value.split("-")
return QueueIdentifier(
to_canonical_address(str_recipient),
self._canonical_id_from_string(str_canonical_id),
)
except (TypeError, ValueError, AttributeError):
raise self.make_error("validator_failed", input=value)
def __init__(self, recipient: Address, channel_identifier: ChannelID,
message_identifier: MessageID) -> None:
# Note that here and only here channel identifier can also be 0 which stands
# for the identifier of no channel (i.e. the global queue)
if not isinstance(channel_identifier, T_ChannelID):
raise ValueError(
"channel identifier must be of type T_ChannelIdentifier")
self.recipient = recipient
self.queue_identifier = QueueIdentifier(
recipient=recipient, channel_identifier=channel_identifier)
self.message_identifier = message_identifier
def test_delivered_message_must_clean_unordered_messages(chain_id):
pseudo_random_generator = random.Random()
block_number = 10
our_address = factories.make_address()
recipient = factories.make_address()
canonical_identifier = factories.make_canonical_identifier()
message_identifier = random.randint(0, 2**16)
secret = factories.random_secret()
chain_state = state.ChainState(
pseudo_random_generator=pseudo_random_generator,
block_number=block_number,
block_hash=factories.make_block_hash(),
our_address=our_address,
chain_id=chain_id,
)
queue_identifier = QueueIdentifier(
recipient=recipient,
canonical_identifier=CANONICAL_IDENTIFIER_GLOBAL_QUEUE)
# Regression test:
# The code delivered_message handler worked only with a queue of one
# element
first_message = SendSecretReveal(
recipient=recipient,
message_identifier=message_identifier,
secret=secret,
canonical_identifier=canonical_identifier,
)
second_message = SendSecretReveal(
recipient=recipient,
message_identifier=random.randint(0, 2**16),
secret=secret,
canonical_identifier=canonical_identifier,
)
chain_state.queueids_to_queues[queue_identifier] = [
first_message, second_message
]
delivered_message = state_change.ReceiveDelivered(recipient,
message_identifier)
iteration = node.handle_receive_delivered(chain_state, delivered_message)
new_queue = iteration.new_state.queueids_to_queues.get(
queue_identifier, [])
assert first_message not in new_queue
def test_retry_queue_does_not_resend_removed_messages(
mock_matrix: MatrixTransport, retry_interval_initial: float) -> None:
"""
Ensure the ``RetryQueue`` doesn't unnecessarily re-send messages.
Messages should only be retried while they are present in the respective Raiden queue.
Once they have been removed they should not be sent again.
In the past they could have been sent twice.
See: https://github.com/raiden-network/raiden/issue/4111
"""
# Pretend the Transport greenlet is running
mock_matrix.greenlet = True
# This is intentionally not using ``MatrixTransport._get_retrier()`` since we don't want the
# greenlet to run but instead manually call its `_check_and_send()` method.
retry_queue = _RetryQueue(transport=mock_matrix,
receiver=Address(factories.HOP1))
message = make_message()
serialized_message = MessageSerializer.serialize(message)
queue_identifier = QueueIdentifier(
recipient=Address(factories.HOP1),
canonical_identifier=CANONICAL_IDENTIFIER_UNORDERED_QUEUE,
)
retry_queue.enqueue(queue_identifier, [message])
# TODO: Fix the code below, the types are not matching.
mock_matrix._queueids_to_queues[queue_identifier] = [message
] # type: ignore
with retry_queue._lock:
retry_queue._check_and_send()
assert len(mock_matrix.sent_messages) == 1 # type: ignore
assert (factories.HOP1,
serialized_message) in mock_matrix.sent_messages # type: ignore
mock_matrix._queueids_to_queues[queue_identifier].clear()
# Make sure the retry interval has elapsed
gevent.sleep(retry_interval_initial * 5)
with retry_queue._lock:
# The message has been removed from the raiden queue and should therefore not be sent again
retry_queue._check_and_send()
assert len(mock_matrix.sent_messages) == 1 # type: ignore
def test_web_rtc_message_sync(matrix_transports):
transport0, transport1 = matrix_transports
transport1_messages = set()
raiden_service0 = MockRaidenService()
raiden_service1 = MockRaidenService()
def mock_handle_web_rtc_messages(message_data, partner_address):
messages = validate_and_parse_message(message_data, partner_address)
transport1_messages.update(messages)
# set mock function to make sure messages are sent via web rtc
transport1._web_rtc_manager._handle_message_callback = mock_handle_web_rtc_messages
transport0.start(raiden_service0, [], None)
transport1.start(raiden_service1, [], None)
transport0.immediate_health_check_for(transport1._raiden_service.address)
transport1.immediate_health_check_for(transport0._raiden_service.address)
with Timeout(TIMEOUT_WEB_RTC_CONNECTION):
# wait until web rtc connection is ready
while not transport0._web_rtc_manager.has_ready_channel(raiden_service1.address):
gevent.sleep(1)
while not transport1._web_rtc_manager.has_ready_channel(raiden_service0.address):
gevent.sleep(1)
queue_identifier = QueueIdentifier(
recipient=transport1._raiden_service.address,
canonical_identifier=factories.UNIT_CANONICAL_ID,
)
raiden0_queues = views.get_all_messagequeues(views.state_from_raiden(raiden_service0))
raiden0_queues[queue_identifier] = []
for i in range(5):
message = Processed(message_identifier=MessageID(i), signature=EMPTY_SIGNATURE)
raiden0_queues[queue_identifier].append(message)
transport0._raiden_service.sign(message)
transport0.send_async([MessagesQueue(queue_identifier, [message])])
with Timeout(TIMEOUT_MESSAGE_RECEIVE):
while not len(transport1_messages) == 5:
gevent.sleep(0.1)
def handle_contract_receive_channel_closed(
chain_state: ChainState, state_change: ContractReceiveChannelClosed
) -> TransitionResult[ChainState]:
# cleanup queue for channel
canonical_identifier = CanonicalIdentifier(
chain_identifier=chain_state.chain_id,
token_network_address=state_change.token_network_address,
channel_identifier=state_change.channel_identifier,
)
channel_state = views.get_channelstate_by_canonical_identifier(
chain_state=chain_state, canonical_identifier=canonical_identifier
)
if channel_state:
queue_id = QueueIdentifier(
recipient=channel_state.partner_state.address,
canonical_identifier=canonical_identifier,
)
if queue_id in chain_state.queueids_to_queues:
chain_state.queueids_to_queues.pop(queue_id)
return handle_token_network_action(chain_state=chain_state, state_change=state_change)
def test_matrix_message_sync(matrix_transports):
transport0, transport1 = matrix_transports
received_messages = set()
message_handler = MessageHandler(received_messages)
raiden_service0 = MockRaidenService(message_handler)
raiden_service1 = MockRaidenService(message_handler)
raiden_service1.handle_and_track_state_change = MagicMock()
transport0.start(raiden_service0, message_handler, None)
transport1.start(raiden_service1, message_handler, None)
gevent.sleep(1)
latest_auth_data = f"{transport1._user_id}/{transport1._client.api.token}"
update_transport_auth_data = ActionUpdateTransportAuthData(
latest_auth_data)
raiden_service1.handle_and_track_state_change.assert_called_with(
update_transport_auth_data)
transport0.start_health_check(transport1._raiden_service.address)
transport1.start_health_check(transport0._raiden_service.address)
queue_identifier = QueueIdentifier(
recipient=transport1._raiden_service.address, channel_identifier=1)
for i in range(5):
message = Processed(message_identifier=i)
transport0._raiden_service.sign(message)
transport0.send_async(queue_identifier, message)
with Timeout(40):
while not len(received_messages) == 10:
gevent.sleep(0.1)
assert len(received_messages) == 10
for i in range(5):
assert any(
getattr(m, "message_identifier", -1) == i
for m in received_messages)
transport1.stop()
assert latest_auth_data
# Send more messages while the other end is offline
for i in range(10, 15):
message = Processed(message_identifier=i)
transport0._raiden_service.sign(message)
transport0.send_async(queue_identifier, message)
# Should fetch the 5 messages sent while transport1 was offline
transport1.start(transport1._raiden_service, message_handler,
latest_auth_data)
gevent.sleep(2)
assert len(set(received_messages)) == 20
for i in range(10, 15):
assert any(
getattr(m, "message_identifier", -1) == i
for m in received_messages)
def test_withdraw_request_message_cleanup(chain_id, token_network_state):
pseudo_random_generator = random.Random()
block_number = 10
our_address = factories.make_address()
recipient1 = factories.make_address()
recipient2 = factories.make_address()
channel_identifier = 1
message_identifier = random.randint(0, 2**16)
chain_state = state.ChainState(
pseudo_random_generator=pseudo_random_generator,
block_number=block_number,
block_hash=factories.make_block_hash(),
our_address=our_address,
chain_id=chain_id,
)
queue_identifier = QueueIdentifier(recipient1,
CANONICAL_IDENTIFIER_GLOBAL_QUEUE)
withdraw_message = SendWithdrawRequest(
message_identifier=message_identifier,
canonical_identifier=CanonicalIdentifier(
chain_identifier=chain_id,
token_network_address=token_network_state.address,
channel_identifier=channel_identifier,
),
total_withdraw=100,
participant=our_address,
recipient=recipient1,
nonce=1,
expiration=10,
)
chain_state.queueids_to_queues[queue_identifier] = [withdraw_message]
processed_message = state_change.ReceiveProcessed(recipient1,
message_identifier)
iteration = node.handle_receive_processed(chain_state, processed_message)
new_queue = iteration.new_state.queueids_to_queues.get(
queue_identifier, [])
# Processed should not have removed the WithdrawRequest message
assert withdraw_message in new_queue
receive_withdraw = ReceiveWithdrawConfirmation(
message_identifier=message_identifier,
canonical_identifier=CanonicalIdentifier(
chain_identifier=chain_id,
token_network_address=token_network_state.address,
channel_identifier=channel_identifier,
),
total_withdraw=100,
signature=factories.make_32bytes(),
sender=recipient2,
participant=recipient2,
nonce=1,
expiration=10,
)
iteration = node.handle_receive_withdraw_confirmation(
chain_state, receive_withdraw)
new_queue = iteration.new_state.queueids_to_queues.get(
queue_identifier, [])
# ReceiveWithdraw from another recipient should not remove the WithdrawRequest
assert withdraw_message in new_queue
receive_withdraw = ReceiveWithdrawConfirmation(
message_identifier=message_identifier,
canonical_identifier=CanonicalIdentifier(
chain_identifier=chain_id,
token_network_address=token_network_state.address,
channel_identifier=channel_identifier,
),
total_withdraw=100,
signature=factories.make_32bytes(),
sender=recipient1,
participant=recipient1,
nonce=1,
expiration=10,
)
iteration = node.handle_receive_withdraw_confirmation(
chain_state, receive_withdraw)
new_queue = iteration.new_state.queueids_to_queues.get(
queue_identifier, [])
assert withdraw_message not in new_queue
def queue_identifier():
return QueueIdentifier(
recipient=factories.make_address(),
canonical_identifier=factories.make_canonical_identifier(),
)
def ping_pong_message_success(transport0, transport1):
queueid0 = QueueIdentifier(
recipient=transport0._raiden_service.address,
canonical_identifier=CANONICAL_IDENTIFIER_UNORDERED_QUEUE,
)
queueid1 = QueueIdentifier(
recipient=transport1._raiden_service.address,
canonical_identifier=CANONICAL_IDENTIFIER_UNORDERED_QUEUE,
)
transport0_raiden_queues = views.get_all_messagequeues(
views.state_from_raiden(transport0._raiden_service)
)
transport1_raiden_queues = views.get_all_messagequeues(
views.state_from_raiden(transport1._raiden_service)
)
transport0_raiden_queues[queueid1] = []
transport1_raiden_queues[queueid0] = []
received_messages0 = transport0._raiden_service.message_handler.bag
received_messages1 = transport1._raiden_service.message_handler.bag
msg_id = random.randint(1e5, 9e5)
ping_message = Processed(message_identifier=msg_id, signature=EMPTY_SIGNATURE)
pong_message = Delivered(delivered_message_identifier=msg_id, signature=EMPTY_SIGNATURE)
transport0_raiden_queues[queueid1].append(ping_message)
transport0._raiden_service.sign(ping_message)
transport1._raiden_service.sign(pong_message)
transport0.send_async([MessagesQueue(queueid1, [ping_message])])
with Timeout(TIMEOUT_MESSAGE_RECEIVE, exception=False):
all_messages_received = False
while not all_messages_received:
all_messages_received = (
ping_message in received_messages1 and pong_message in received_messages0
)
gevent.sleep(0.1)
assert ping_message in received_messages1
assert pong_message in received_messages0
transport0_raiden_queues[queueid1].clear()
transport1_raiden_queues[queueid0].append(ping_message)
transport0._raiden_service.sign(pong_message)
transport1._raiden_service.sign(ping_message)
transport1.send_async([MessagesQueue(queueid0, [ping_message])])
with Timeout(TIMEOUT_MESSAGE_RECEIVE, exception=False):
all_messages_received = False
while not all_messages_received:
all_messages_received = (
ping_message in received_messages0 and pong_message in received_messages1
)
gevent.sleep(0.1)
assert ping_message in received_messages0
assert pong_message in received_messages1
transport1_raiden_queues[queueid0].clear()
return all_messages_received
def test_matrix_message_sync(matrix_transports):
transport0, transport1 = matrix_transports
transport0_messages = set()
transport1_messages = set()
transport0_message_handler = MessageHandler(transport0_messages)
transport1_message_handler = MessageHandler(transport1_messages)
raiden_service0 = MockRaidenService(transport0_message_handler)
raiden_service1 = MockRaidenService(transport1_message_handler)
raiden_service1.handle_and_track_state_changes = MagicMock()
transport0.start(raiden_service0, [], None)
transport1.start(raiden_service1, [], None)
transport0.immediate_health_check_for(transport1._raiden_service.address)
transport1.immediate_health_check_for(transport0._raiden_service.address)
queue_identifier = QueueIdentifier(
recipient=transport1._raiden_service.address,
canonical_identifier=factories.UNIT_CANONICAL_ID,
)
raiden0_queues = views.get_all_messagequeues(views.state_from_raiden(raiden_service0))
raiden0_queues[queue_identifier] = []
for i in range(5):
message = Processed(message_identifier=i, signature=EMPTY_SIGNATURE)
raiden0_queues[queue_identifier].append(message)
transport0._raiden_service.sign(message)
transport0.send_async([MessagesQueue(queue_identifier, [message])])
with Timeout(TIMEOUT_MESSAGE_RECEIVE):
while not len(transport0_messages) == 5:
gevent.sleep(0.1)
while not len(transport1_messages) == 5:
gevent.sleep(0.1)
# transport1 receives the `Processed` messages sent by transport0
for i in range(5):
assert any(m.message_identifier == i for m in transport1_messages)
# transport0 answers with a `Delivered` for each `Processed`
for i in range(5):
assert any(m.delivered_message_identifier == i for m in transport0_messages)
# Clear out queue
raiden0_queues[queue_identifier] = []
transport1.stop()
wait_for_peer_unreachable(transport0, transport1._raiden_service.address)
# Send more messages while the other end is offline
for i in range(10, 15):
message = Processed(message_identifier=i, signature=EMPTY_SIGNATURE)
raiden0_queues[queue_identifier].append(message)
transport0._raiden_service.sign(message)
transport0.send_async([MessagesQueue(queue_identifier, [message])])
# Should fetch the 5 messages sent while transport1 was offline
transport1.start(transport1._raiden_service, [], None)
transport1.immediate_health_check_for(transport0._raiden_service.address)
with gevent.Timeout(TIMEOUT_MESSAGE_RECEIVE):
while len(transport1_messages) != 10:
gevent.sleep(0.1)
while len(transport0_messages) != 10:
gevent.sleep(0.1)
# transport1 receives the 5 new `Processed` messages sent by transport0
for i in range(10, 15):
assert any(m.message_identifier == i for m in transport1_messages)
# transport0 answers with a `Delivered` for each one of the new `Processed`
for i in range(10, 15):
assert any(m.delivered_message_identifier == i for m in transport0_messages)
def _deserialize(self, queue_identifier_str: str, attr: Any, data: Any,
**kwargs: Any) -> QueueIdentifier:
str_recipient, str_canonical_id = queue_identifier_str.split("-")
return QueueIdentifier(
to_canonical_address(str_recipient),
self._canonical_id_from_string(str_canonical_id))
def test_matrix_message_retry(
local_matrix_servers,
retry_interval_initial,
retry_interval_max,
retries_before_backoff,
broadcast_rooms,
):
""" Test the retry mechanism implemented into the matrix client.
The test creates a transport and sends a message. Given that the
receiver was online, the initial message is sent but the receiver
doesn't respond in time and goes offline. The retrier should then
wait for the `retry_interval` duration to pass and send the message
again but this won't work because the receiver is offline. Once
the receiver comes back again, the message should be sent again.
"""
partner_address = factories.make_address()
transport = MatrixTransport(
config=MatrixTransportConfig(
broadcast_rooms=broadcast_rooms,
retries_before_backoff=retries_before_backoff,
retry_interval_initial=retry_interval_initial,
retry_interval_max=retry_interval_max,
server=local_matrix_servers[0],
available_servers=[local_matrix_servers[0]],
),
environment=Environment.DEVELOPMENT,
)
transport._send_raw = MagicMock()
raiden_service = MockRaidenService(None)
transport.start(raiden_service, [], None)
transport.log = MagicMock()
# Receiver is online
transport._address_mgr._address_to_reachabilitystate[partner_address] = ReachabilityState(
AddressReachability.REACHABLE, datetime.now()
)
queueid = QueueIdentifier(
recipient=partner_address, canonical_identifier=CANONICAL_IDENTIFIER_UNORDERED_QUEUE
)
chain_state = raiden_service.wal.state_manager.current_state
retry_queue: _RetryQueue = transport._get_retrier(partner_address)
assert bool(retry_queue), "retry_queue not running"
# Send the initial message
message = Processed(message_identifier=0, signature=EMPTY_SIGNATURE)
transport._raiden_service.sign(message)
chain_state.queueids_to_queues[queueid] = [message]
retry_queue.enqueue_unordered(message)
gevent.idle()
assert transport._send_raw.call_count == 1
# Receiver goes offline
transport._address_mgr._address_to_reachabilitystate[partner_address] = ReachabilityState(
AddressReachability.UNREACHABLE, datetime.now()
)
with gevent.Timeout(retry_interval_initial + 2):
wait_assert(
transport.log.debug.assert_called_with,
"Partner not reachable. Skipping.",
partner=to_checksum_address(partner_address),
status=AddressReachability.UNREACHABLE,
)
# Retrier did not call send_raw given that the receiver is still offline
assert transport._send_raw.call_count == 1
# Receiver comes back online
transport._address_mgr._address_to_reachabilitystate[partner_address] = ReachabilityState(
AddressReachability.REACHABLE, datetime.now()
)
# Retrier should send the message again
with gevent.Timeout(retry_interval_initial + 2):
while transport._send_raw.call_count != 2:
gevent.sleep(0.1)
transport.stop()
transport.greenlet.get()
def test_matrix_message_retry(local_matrix_servers, private_rooms,
retry_interval, retries_before_backoff,
global_rooms):
""" Test the retry mechanism implemented into the matrix client.
The test creates a transport and sends a message. Given that the
receiver was online, the initial message is sent but the receiver
doesn't respond in time and goes offline. The retrier should then
wait for the `retry_interval` duration to pass and send the message
again but this won't work because the receiver is offline. Once
the receiver comes back again, the message should be sent again.
"""
partner_address = factories.make_address()
transport = MatrixTransport({
"global_rooms": global_rooms,
"retries_before_backoff": retries_before_backoff,
"retry_interval": retry_interval,
"server": local_matrix_servers[0],
"server_name": local_matrix_servers[0].netloc,
"available_servers": [local_matrix_servers[0]],
"private_rooms": private_rooms,
})
transport._send_raw = MagicMock()
raiden_service = MockRaidenService(None)
transport.start(raiden_service, raiden_service.message_handler, None)
transport.log = MagicMock()
# Receiver is online
transport._address_mgr._address_to_reachability[
partner_address] = AddressReachability.REACHABLE
queueid = QueueIdentifier(
recipient=partner_address,
channel_identifier=CHANNEL_IDENTIFIER_GLOBAL_QUEUE)
chain_state = raiden_service.wal.state_manager.current_state
retry_queue: _RetryQueue = transport._get_retrier(partner_address)
assert bool(retry_queue), "retry_queue not running"
# Send the initial message
message = Processed(message_identifier=0)
transport._raiden_service.sign(message)
chain_state.queueids_to_queues[queueid] = [message]
retry_queue.enqueue_global(message)
gevent.sleep(1)
assert transport._send_raw.call_count == 1
# Receiver goes offline
transport._address_mgr._address_to_reachability[
partner_address] = AddressReachability.UNREACHABLE
gevent.sleep(retry_interval)
transport.log.debug.assert_called_with(
"Partner not reachable. Skipping.",
partner=pex(partner_address),
status=AddressReachability.UNREACHABLE,
)
# Retrier did not call send_raw given that the receiver is still offline
assert transport._send_raw.call_count == 1
# Receiver comes back online
transport._address_mgr._address_to_reachability[
partner_address] = AddressReachability.REACHABLE
gevent.sleep(retry_interval)
# Retrier now should have sent the message again
assert transport._send_raw.call_count == 2
transport.stop()
transport.get()
def __post_init__(self) -> None:
queue_identifier = QueueIdentifier(
recipient=self.recipient, canonical_identifier=self.canonical_identifier
)
object.__setattr__(self, "queue_identifier", queue_identifier)
def test_matrix_message_sync(matrix_transports):
transport0, transport1 = matrix_transports
received_messages = set()
message_handler = MessageHandler(received_messages)
raiden_service0 = MockRaidenService(message_handler)
raiden_service1 = MockRaidenService(message_handler)
raiden_service1.handle_and_track_state_changes = MagicMock()
transport0.start(raiden_service0, message_handler, None)
transport1.start(raiden_service1, message_handler, None)
latest_auth_data = f"{transport1._user_id}/{transport1._client.api.token}"
update_transport_auth_data = ActionUpdateTransportAuthData(
latest_auth_data)
with gevent.Timeout(2):
wait_assert(
raiden_service1.handle_and_track_state_changes.assert_called_with,
[update_transport_auth_data],
)
transport0.start_health_check(transport1._raiden_service.address)
transport1.start_health_check(transport0._raiden_service.address)
queue_identifier = QueueIdentifier(
recipient=transport1._raiden_service.address,
canonical_identifier=factories.UNIT_CANONICAL_ID,
)
raiden0_queues = views.get_all_messagequeues(
views.state_from_raiden(raiden_service0))
raiden0_queues[queue_identifier] = []
for i in range(5):
message = Processed(message_identifier=i, signature=EMPTY_SIGNATURE)
raiden0_queues[queue_identifier].append(message)
transport0._raiden_service.sign(message)
transport0.send_async(queue_identifier, message)
with Timeout(TIMEOUT_MESSAGE_RECEIVE):
while not len(received_messages) == 10:
gevent.sleep(0.1)
assert len(received_messages) == 10
for i in range(5):
assert any(
getattr(m, "message_identifier", -1) == i
for m in received_messages)
# Clear out queue
raiden0_queues[queue_identifier] = []
transport1.stop()
wait_for_peer_unreachable(transport0, transport1._raiden_service.address)
assert latest_auth_data
# Send more messages while the other end is offline
for i in range(10, 15):
message = Processed(message_identifier=i, signature=EMPTY_SIGNATURE)
raiden0_queues[queue_identifier].append(message)
transport0._raiden_service.sign(message)
transport0.send_async(queue_identifier, message)
# Should fetch the 5 messages sent while transport1 was offline
transport1.start(transport1._raiden_service, message_handler,
latest_auth_data)
transport1.start_health_check(transport0._raiden_service.address)
with gevent.Timeout(TIMEOUT_MESSAGE_RECEIVE):
while len(set(received_messages)) != 20:
gevent.sleep(0.1)
assert len(set(received_messages)) == 20
for i in range(10, 15):
assert any(
getattr(m, "message_identifier", -1) == i
for m in received_messages)
