def test_matrix_discovery_room_offline_server(
local_matrix_servers,
retries_before_backoff,
retry_interval,
private_rooms,
):
transport = MatrixTransport({
'discovery_room':
'discovery',
'retries_before_backoff':
retries_before_backoff,
'retry_interval':
retry_interval,
'server':
local_matrix_servers[0],
'server_name':
local_matrix_servers[0].netloc,
'available_servers': [local_matrix_servers[0], 'https://localhost:1'],
'private_rooms':
private_rooms,
})
transport.start(MockRaidenService(None), MessageHandler(set()), '')
gevent.sleep(.2)
transport.stop()
transport.get()
def test_join_invalid_discovery(
skip_if_not_matrix,
local_matrix_server,
private_rooms,
retry_interval,
retries_before_backoff,
):
"""_join_discovery_room tries to join on all servers on available_servers config
If any of the servers isn't reachable by synapse, it'll return a 500 response, which needs
to be handled, and if no discovery room is found on any of the available_servers, one in
our current server should be created
"""
transport = MatrixTransport({
'discovery_room': 'discovery',
'retries_before_backoff': retries_before_backoff,
'retry_interval': retry_interval,
'server': local_matrix_server,
'server_name': 'matrix.local.raiden',
'available_servers': ['http://invalid.server'],
'private_rooms': private_rooms,
})
transport._client.api.retry_timeout = 0
transport._send_raw = MagicMock()
raiden_service = MockRaidenService(None)
transport.start(
raiden_service,
raiden_service.message_handler,
None,
)
transport.log = MagicMock()
transport._join_discovery_room()
assert isinstance(transport._discovery_room, Room)
transport.stop()
transport.get()
def test_send_queued_messages( # pylint: disable=unused-argument
raiden_network, deposit, token_addresses, network_wait):
"""Test re-sending of undelivered messages on node restart"""
app0, app1 = raiden_network
token_address = token_addresses[0]
chain_state = views.state_from_app(app0)
token_network_registry_address = app0.raiden.default_registry.address
token_network_address = views.get_token_network_address_by_token_address(
chain_state, token_network_registry_address, token_address)
assert token_network_address
number_of_transfers = 7
amount_per_transfer = 1
total_transferred_amount = TokenAmount(amount_per_transfer *
number_of_transfers)
# Make sure none of the transfers will be sent before the restart
transfers = []
for secret_seed in range(number_of_transfers):
secret = make_secret(secret_seed)
secrethash = sha256_secrethash(secret)
transfers.append((create_default_identifier(), amount_per_transfer,
secret, secrethash))
app0.raiden.raiden_event_handler.hold(
SendLockedTransfer,
{"transfer": {
"lock": {
"secrethash": secrethash
}
}})
for identifier, amount, secret, _ in transfers:
app0.raiden.mediated_transfer_async(
token_network_address=token_network_address,
amount=amount,
target=app1.raiden.address,
identifier=identifier,
secret=secret,
)
app0.stop()
# Restart the app. The pending transfers must be processed.
new_transport = MatrixTransport(app0.raiden.config["transport"]["matrix"])
raiden_event_handler = RaidenEventHandler()
message_handler = MessageHandler()
app0_restart = App(
config=app0.config,
rpc_client=app0.raiden.rpc_client,
proxy_manager=app0.raiden.proxy_manager,
query_start_block=BlockNumber(0),
default_registry=app0.raiden.default_registry,
default_secret_registry=app0.raiden.default_secret_registry,
default_service_registry=app0.raiden.default_service_registry,
default_one_to_n_address=app0.raiden.default_one_to_n_address,
default_msc_address=app0.raiden.default_msc_address,
transport=new_transport,
raiden_event_handler=raiden_event_handler,
message_handler=message_handler,
routing_mode=RoutingMode.PRIVATE,
)
del app0
app0_restart.start()
# XXX: There is no synchronization among the app and the test, so it is
# possible between `start` and the check bellow that some of the transfers
# have completed, making it flaky.
#
# Make sure the transfers are in the queue and fail otherwise.
# chain_state = views.state_from_raiden(app0_restart.raiden)
# for _, _, _, secrethash in transfers:
# msg = "The secrethashes of the pending transfers must be in the queue after a restart."
# assert secrethash in chain_state.payment_mapping.secrethashes_to_task, msg
with watch_for_unlock_failures(*raiden_network):
exception = RuntimeError(
"Timeout while waiting for balance update for app0")
with gevent.Timeout(20, exception=exception):
waiting.wait_for_payment_balance(
raiden=app0_restart.raiden,
token_network_registry_address=token_network_registry_address,
token_address=token_address,
partner_address=app1.raiden.address,
target_address=app1.raiden.address,
target_balance=total_transferred_amount,
retry_timeout=network_wait,
)
exception = RuntimeError(
"Timeout while waiting for balance update for app1")
with gevent.Timeout(20, exception=exception):
waiting.wait_for_payment_balance(
raiden=app1.raiden,
token_network_registry_address=token_network_registry_address,
token_address=token_address,
partner_address=app0_restart.raiden.address,
target_address=app1.raiden.address,
target_balance=total_transferred_amount,
retry_timeout=network_wait,
)
assert_synced_channel_state(
token_network_address,
app0_restart,
deposit - total_transferred_amount,
[],
app1,
deposit + total_transferred_amount,
[],
)
new_transport.stop()
def test_matrix_message_retry(
local_matrix_servers,
private_rooms,
retry_interval,
retries_before_backoff,
):
""" Test the retry mechanism implemented into the matrix client.
The test creates a transport and sends a message. Given that the
receiver was online, the initial message is sent but the receiver
doesn't respond in time and goes offline. The retrier should then
wait for the `retry_interval` duration to pass and send the message
again but this won't work because the receiver is offline. Once
the receiver comes back again, the message should be sent again.
"""
partner_address = make_address()
transport = MatrixTransport({
'discovery_room': 'discovery',
'retries_before_backoff': retries_before_backoff,
'retry_interval': retry_interval,
'server': local_matrix_servers[0],
'server_name': local_matrix_servers[0].netloc,
'available_servers': [],
'private_rooms': private_rooms,
})
transport._send_raw = MagicMock()
raiden_service = MockRaidenService(None)
transport.start(
raiden_service,
raiden_service.message_handler,
None,
)
transport.log = MagicMock()
# Receiver is online
transport._address_to_presence[partner_address] = UserPresence.ONLINE
queueid = QueueIdentifier(
recipient=partner_address,
channel_identifier=CHANNEL_IDENTIFIER_GLOBAL_QUEUE,
)
chain_state = raiden_service.wal.state_manager.current_state
retry_queue: _RetryQueue = transport._get_retrier(partner_address)
assert bool(retry_queue), 'retry_queue not running'
# Send the initial message
message = Processed(0)
message.sign(transport._raiden_service.private_key)
chain_state.queueids_to_queues[queueid] = [message]
retry_queue.enqueue_global(message)
gevent.sleep(1)
transport._send_raw.call_count = 1
# Receiver goes offline
transport._address_to_presence[partner_address] = UserPresence.OFFLINE
gevent.sleep(retry_interval)
transport.log.debug.assert_called_with(
'Partner not reachable. Skipping.',
partner=pex(partner_address),
status=UserPresence.OFFLINE,
)
# Retrier did not call send_raw given that the receiver is still offline
assert transport._send_raw.call_count == 1
# Receiver comes back online
transport._address_to_presence[partner_address] = UserPresence.ONLINE
gevent.sleep(retry_interval)
# Retrier now should have sent the message again
assert transport._send_raw.call_count == 2
transport.stop()
transport.get()
def test_matrix_message_sync(
local_matrix_servers,
private_rooms,
retry_interval,
retries_before_backoff,
):
transport0 = MatrixTransport({
'discovery_room': 'discovery',
'retries_before_backoff': retries_before_backoff,
'retry_interval': retry_interval,
'server': local_matrix_servers[0],
'server_name': local_matrix_servers[0].netloc,
'available_servers': [],
'private_rooms': private_rooms,
})
transport1 = MatrixTransport({
'discovery_room': 'discovery',
'retries_before_backoff': retries_before_backoff,
'retry_interval': retry_interval,
'server': local_matrix_servers[0],
'server_name': local_matrix_servers[0].netloc,
'available_servers': [],
'private_rooms': private_rooms,
})
latest_auth_data = None
received_messages = set()
message_handler = MessageHandler(received_messages)
raiden_service0 = MockRaidenService(message_handler)
raiden_service1 = MockRaidenService(message_handler)
raiden_service1.handle_state_change = MagicMock()
transport0.start(
raiden_service0,
message_handler,
None,
)
transport1.start(
raiden_service1,
message_handler,
None,
)
gevent.sleep(1)
latest_auth_data = f'{transport1._user_id}/{transport1._client.api.token}'
update_transport_auth_data = ActionUpdateTransportAuthData(latest_auth_data)
raiden_service1.handle_state_change.assert_called_with(update_transport_auth_data)
transport0.start_health_check(transport1._raiden_service.address)
transport1.start_health_check(transport0._raiden_service.address)
queue_identifier = QueueIdentifier(
recipient=transport1._raiden_service.address,
channel_identifier=1,
)
for i in range(5):
message = Processed(i)
message.sign(transport0._raiden_service.private_key)
transport0.send_async(
queue_identifier,
message,
)
gevent.sleep(2)
assert len(received_messages) == 10
for i in range(5):
assert any(getattr(m, 'message_identifier', -1) == i for m in received_messages)
transport1.stop()
assert latest_auth_data
# Send more messages while the other end is offline
for i in range(10, 15):
message = Processed(i)
message.sign(transport0._raiden_service.private_key)
transport0.send_async(
queue_identifier,
message,
)
# Should fetch the 5 messages sent while transport1 was offline
transport1.start(
transport1._raiden_service,
message_handler,
latest_auth_data,
)
gevent.sleep(2)
assert len(set(received_messages)) == 20
for i in range(10, 15):
assert any(getattr(m, 'message_identifier', -1) == i for m in received_messages)
transport0.stop()
transport1.stop()
transport0.get()
transport1.get()
def test_send_queued_messages_after_restart( # pylint: disable=unused-argument
raiden_network: List[RaidenService],
restart_node: RestartNode,
deposit: TokenAmount,
token_addresses: List[TokenAddress],
network_wait: float,
):
"""Test re-sending of undelivered messages on node restart"""
app0, app1 = raiden_network
token_address = token_addresses[0]
chain_state = views.state_from_raiden(app0)
token_network_registry_address = app0.default_registry.address
token_network_address = views.get_token_network_address_by_token_address(
chain_state, token_network_registry_address, token_address)
assert token_network_address
number_of_transfers = 7
amount_per_transfer = PaymentAmount(1)
total_transferred_amount = TokenAmount(amount_per_transfer *
number_of_transfers)
# Make sure none of the transfers will be sent before the restart
transfers = []
for secret_seed in range(number_of_transfers):
secret = make_secret(secret_seed)
secrethash = sha256_secrethash(secret)
transfers.append((create_default_identifier(), amount_per_transfer,
secret, secrethash))
assert isinstance(app0.raiden_event_handler,
HoldRaidenEventHandler) # for mypy
app0.raiden_event_handler.hold(
SendLockedTransfer,
{"transfer": {
"lock": {
"secrethash": secrethash
}
}})
for identifier, amount, secret, _ in transfers:
app0.mediated_transfer_async(
token_network_address=token_network_address,
amount=amount,
target=TargetAddress(app1.address),
identifier=identifier,
secret=secret,
)
app0.stop()
# Restart the app. The pending transfers must be processed.
new_transport = MatrixTransport(config=app0.config.transport,
environment=app0.config.environment_type)
raiden_event_handler = RaidenEventHandler()
message_handler = MessageHandler()
app0_restart = RaidenService(
config=app0.config,
rpc_client=app0.rpc_client,
proxy_manager=app0.proxy_manager,
query_start_block=BlockNumber(0),
raiden_bundle=RaidenBundle(
app0.default_registry,
app0.default_secret_registry,
),
services_bundle=app0.default_services_bundle,
transport=new_transport,
raiden_event_handler=raiden_event_handler,
message_handler=message_handler,
routing_mode=RoutingMode.PRIVATE,
)
del app0
restart_node(app0_restart)
# XXX: There is no synchronization among the app and the test, so it is
# possible between `start` and the check below that some of the transfers
# have completed, making it flaky.
#
# Make sure the transfers are in the queue and fail otherwise.
chain_state = views.state_from_raiden(app0_restart)
for _, _, _, secrethash in transfers:
msg = "The secrethashes of the pending transfers must be in the queue after a restart."
assert secrethash in chain_state.payment_mapping.secrethashes_to_task, msg
timeout = block_offset_timeout(
app1, "Timeout waiting for balance update of app0")
with watch_for_unlock_failures(*raiden_network), timeout:
waiting.wait_for_payment_balance(
raiden=app0_restart,
token_network_registry_address=token_network_registry_address,
token_address=token_address,
partner_address=app1.address,
target_address=app1.address,
target_balance=total_transferred_amount,
retry_timeout=network_wait,
)
timeout.exception_to_throw = ValueError(
"Timeout waiting for balance update of app1")
waiting.wait_for_payment_balance(
raiden=app1,
token_network_registry_address=token_network_registry_address,
token_address=token_address,
partner_address=app0_restart.address,
target_address=app1.address,
target_balance=total_transferred_amount,
retry_timeout=network_wait,
)
assert_synced_channel_state(
token_network_address,
app0_restart,
Balance(deposit - total_transferred_amount),
[],
app1,
Balance(deposit + total_transferred_amount),
[],
)
new_transport.stop()
def test_matrix_message_sync(
skip_if_not_matrix,
local_matrix_server,
private_rooms,
retry_interval,
retries_before_backoff,
):
transport0 = MatrixTransport({
'discovery_room': 'discovery',
'retries_before_backoff': retries_before_backoff,
'retry_interval': retry_interval,
'server': local_matrix_server,
'server_name': 'matrix.local.raiden',
'available_servers': [],
'private_rooms': private_rooms,
})
transport1 = MatrixTransport({
'discovery_room': 'discovery',
'retries_before_backoff': retries_before_backoff,
'retry_interval': retry_interval,
'server': local_matrix_server,
'server_name': 'matrix.local.raiden',
'available_servers': [],
'private_rooms': private_rooms,
})
latest_sync_token = None
received_messages = set()
def hook(sync_token):
nonlocal latest_sync_token
latest_sync_token = sync_token
class MessageHandler:
def on_message(self, _, message):
nonlocal received_messages
received_messages.add(message)
transport0._client.set_post_sync_hook(hook)
message_handler = MessageHandler()
raiden_service0 = MockRaidenService(message_handler)
raiden_service1 = MockRaidenService(message_handler)
raiden_service1.handle_state_change = MagicMock()
transport0.start(
raiden_service0,
message_handler,
None,
)
transport1.start(
raiden_service1,
message_handler,
None,
)
transport0.start_health_check(transport1._raiden_service.address)
transport1.start_health_check(transport0._raiden_service.address)
queue_identifier = QueueIdentifier(
recipient=transport1._raiden_service.address,
channel_identifier=1,
)
for i in range(5):
message = Processed(i)
message.sign(transport0._raiden_service.private_key)
transport0.send_async(
queue_identifier,
message,
)
gevent.sleep(2)
latest_sync_token = f'{transport1._user_id}/{latest_sync_token}'
update_transport_sync_token = ActionUpdateTransportSyncToken(latest_sync_token)
raiden_service1.handle_state_change.assert_called_with(update_transport_sync_token)
assert len(received_messages) == 10
for i in range(5):
assert any(getattr(m, 'message_identifier', -1) == i for m in received_messages)
transport1.stop()
assert latest_sync_token
# Send more messages while the other end is offline
for i in range(10, 15):
message = Processed(i)
message.sign(transport0._raiden_service.private_key)
transport0.send_async(
queue_identifier,
message,
)
# Should fetch the 5 messages sent while transport1 was offline
transport1.start(
transport1._raiden_service,
message_handler,
latest_sync_token,
)
gevent.sleep(2)
assert len(set(received_messages)) == 20
for i in range(10, 15):
assert any(getattr(m, 'message_identifier', -1) == i for m in received_messages)
transport0.stop()
transport1.stop()
