async def _unsafe_process(self):
# If self.pos is None then means we haven't fetched it from DB
if self.pos is None:
self.pos = await self.store.get_stats_positions()
# Loop round handling deltas until we're up to date
while True:
# Be sure to read the max stream_ordering *before* checking if there are any outstanding
# deltas, since there is otherwise a chance that we could miss updates which arrive
# after we check the deltas.
room_max_stream_ordering = self.store.get_room_max_stream_ordering(
)
if self.pos == room_max_stream_ordering:
break
logger.debug("Processing room stats %s->%s", self.pos,
room_max_stream_ordering)
max_pos, deltas = await self.store.get_current_state_deltas(
self.pos, room_max_stream_ordering)
if deltas:
logger.debug("Handling %d state deltas", len(deltas))
room_deltas, user_deltas = await self._handle_deltas(deltas)
else:
room_deltas = {}
user_deltas = {}
# Then count deltas for total_events and total_event_bytes.
(
room_count,
user_count,
) = await self.store.get_changes_room_total_events_and_bytes(
self.pos, max_pos)
for room_id, fields in room_count.items():
room_deltas.setdefault(room_id, {}).update(fields)
for user_id, fields in user_count.items():
user_deltas.setdefault(user_id, {}).update(fields)
logger.debug("room_deltas: %s", room_deltas)
logger.debug("user_deltas: %s", user_deltas)
# Always call this so that we update the stats position.
await self.store.bulk_update_stats_delta(
self.clock.time_msec(),
updates={
"room": room_deltas,
"user": user_deltas
},
stream_id=max_pos,
)
logger.debug("Handled room stats to %s -> %s", self.pos, max_pos)
event_processing_positions.labels("stats").set(max_pos)
self.pos = max_pos
async def _unsafe_process(self) -> None:
# If self.pos is None then means we haven't fetched it from DB
if self.pos is None:
self.pos = await self.store.get_stats_positions()
room_max_stream_ordering = self.store.get_room_max_stream_ordering(
)
if self.pos > room_max_stream_ordering:
# apparently, we've processed more events than exist in the database!
# this can happen if events are removed with history purge or similar.
logger.warning(
"Event stream ordering appears to have gone backwards (%i -> %i): "
"rewinding stats processor",
self.pos,
room_max_stream_ordering,
)
self.pos = room_max_stream_ordering
# Loop round handling deltas until we're up to date
while True:
# Be sure to read the max stream_ordering *before* checking if there are any outstanding
# deltas, since there is otherwise a chance that we could miss updates which arrive
# after we check the deltas.
room_max_stream_ordering = self.store.get_room_max_stream_ordering(
)
if self.pos == room_max_stream_ordering:
break
logger.debug("Processing room stats %s->%s", self.pos,
room_max_stream_ordering)
max_pos, deltas = await self.store.get_current_state_deltas(
self.pos, room_max_stream_ordering)
if deltas:
logger.debug("Handling %d state deltas", len(deltas))
room_deltas, user_deltas = await self._handle_deltas(deltas)
else:
room_deltas = {}
user_deltas = {}
logger.debug("room_deltas: %s", room_deltas)
logger.debug("user_deltas: %s", user_deltas)
# Always call this so that we update the stats position.
await self.store.bulk_update_stats_delta(
self.clock.time_msec(),
updates={
"room": room_deltas,
"user": user_deltas
},
stream_id=max_pos,
)
logger.debug("Handled room stats to %s -> %s", self.pos, max_pos)
event_processing_positions.labels("stats").set(max_pos)
self.pos = max_pos
def _unsafe_process(self):
# If self.pos is None then means we haven't fetched it from DB
if self.pos is None:
self.pos = yield self.store.get_stats_stream_pos()
# If still None then the initial background update hasn't happened yet
if self.pos is None:
defer.returnValue(None)
# Loop round handling deltas until we're up to date
while True:
with Measure(self.clock, "stats_delta"):
deltas = yield self.store.get_current_state_deltas(self.pos)
if not deltas:
return
logger.info("Handling %d state deltas", len(deltas))
yield self._handle_deltas(deltas)
self.pos = deltas[-1]["stream_id"]
yield self.store.update_stats_stream_pos(self.pos)
event_processing_positions.labels("stats").set(self.pos)
