def test_linearizer_is_queued(self) -> None:
"""Tests `Linearizer.is_queued`.
Runs through the same scenario as `test_linearizer`.
"""
linearizer = Linearizer()
key = object()
_, acquired_d1, unblock1 = self._start_task(linearizer, key)
self.assertTrue(acquired_d1.called)
# Since the first task acquires the lock immediately, "is_queued" should return
# false.
self.assertFalse(linearizer.is_queued(key))
_, acquired_d2, unblock2 = self._start_task(linearizer, key)
self.assertFalse(acquired_d2.called)
# Now the second task is queued up behind the first.
self.assertTrue(linearizer.is_queued(key))
unblock1()
# And now the second task acquires the lock and nothing is in the queue again.
self.assertTrue(acquired_d2.called)
self.assertFalse(linearizer.is_queued(key))
unblock2()
self.assertFalse(linearizer.is_queued(key))
def test_linearizer_is_queued(self):
linearizer = Linearizer()
key = object()
d1 = linearizer.queue(key)
cm1 = yield d1
# Since d1 gets called immediately, "is_queued" should return false.
self.assertFalse(linearizer.is_queued(key))
d2 = linearizer.queue(key)
self.assertFalse(d2.called)
# Now d2 is queued up behind successful completion of cm1
self.assertTrue(linearizer.is_queued(key))
with cm1:
self.assertFalse(d2.called)
# cm1 still not done, so d2 still queued.
self.assertTrue(linearizer.is_queued(key))
# And now d2 is called and nothing is in the queue again
self.assertFalse(linearizer.is_queued(key))
with (yield d2):
self.assertFalse(linearizer.is_queued(key))
self.assertFalse(linearizer.is_queued(key))
class FederationSenderHandler:
"""Processes the fedration replication stream
This class is only instantiate on the worker responsible for sending outbound
federation transactions. It receives rows from the replication stream and forwards
the appropriate entries to the FederationSender class.
"""
def __init__(self, hs: GenericWorkerServer):
self.store = hs.get_datastore()
self._is_mine_id = hs.is_mine_id
self.federation_sender = hs.get_federation_sender()
self._hs = hs
# Stores the latest position in the federation stream we've gotten up
# to. This is always set before we use it.
self.federation_position = None
self._fed_position_linearizer = Linearizer(
name="_fed_position_linearizer")
def on_start(self):
# There may be some events that are persisted but haven't been sent,
# so send them now.
self.federation_sender.notify_new_events(
self.store.get_room_max_stream_ordering())
def wake_destination(self, server: str):
self.federation_sender.wake_destination(server)
async def process_replication_rows(self, stream_name, token, rows):
# The federation stream contains things that we want to send out, e.g.
# presence, typing, etc.
if stream_name == "federation":
send_queue.process_rows_for_federation(self.federation_sender,
rows)
await self.update_token(token)
# ... and when new receipts happen
elif stream_name == ReceiptsStream.NAME:
await self._on_new_receipts(rows)
# ... as well as device updates and messages
elif stream_name == DeviceListsStream.NAME:
# The entities are either user IDs (starting with '@') whose devices
# have changed, or remote servers that we need to tell about
# changes.
hosts = {
row.entity
for row in rows if not row.entity.startswith("@")
}
for host in hosts:
self.federation_sender.send_device_messages(host)
elif stream_name == ToDeviceStream.NAME:
# The to_device stream includes stuff to be pushed to both local
# clients and remote servers, so we ignore entities that start with
# '@' (since they'll be local users rather than destinations).
hosts = {
row.entity
for row in rows if not row.entity.startswith("@")
}
for host in hosts:
self.federation_sender.send_device_messages(host)
async def _on_new_receipts(self, rows):
"""
Args:
rows (Iterable[synapse.replication.tcp.streams.ReceiptsStream.ReceiptsStreamRow]):
new receipts to be processed
"""
for receipt in rows:
# we only want to send on receipts for our own users
if not self._is_mine_id(receipt.user_id):
continue
receipt_info = ReadReceipt(
receipt.room_id,
receipt.receipt_type,
receipt.user_id,
[receipt.event_id],
receipt.data,
)
await self.federation_sender.send_read_receipt(receipt_info)
async def update_token(self, token):
"""Update the record of where we have processed to in the federation stream.
Called after we have processed a an update received over replication. Sends
a FEDERATION_ACK back to the master, and stores the token that we have processed
in `federation_stream_position` so that we can restart where we left off.
"""
self.federation_position = token
# We save and send the ACK to master asynchronously, so we don't block
# processing on persistence. We don't need to do this operation for
# every single RDATA we receive, we just need to do it periodically.
if self._fed_position_linearizer.is_queued(None):
# There is already a task queued up to save and send the token, so
# no need to queue up another task.
return
run_as_background_process("_save_and_send_ack",
self._save_and_send_ack)
async def _save_and_send_ack(self):
"""Save the current federation position in the database and send an ACK
to master with where we're up to.
"""
try:
# We linearize here to ensure we don't have races updating the token
#
# XXX this appears to be redundant, since the ReplicationCommandHandler
# has a linearizer which ensures that we only process one line of
# replication data at a time. Should we remove it, or is it doing useful
# service for robustness? Or could we replace it with an assertion that
# we're not being re-entered?
with (await self._fed_position_linearizer.queue(None)):
# We persist and ack the same position, so we take a copy of it
# here as otherwise it can get modified from underneath us.
current_position = self.federation_position
await self.store.update_federation_out_pos(
"federation", current_position)
# We ACK this token over replication so that the master can drop
# its in memory queues
self._hs.get_tcp_replication().send_federation_ack(
current_position)
except Exception:
logger.exception("Error updating federation stream position")
