async def _unsafe_process(self): # If self.pos is None then means we haven't fetched it from DB if self.pos is None: self.pos = await self.store.get_stats_positions() # Loop round handling deltas until we're up to date while True: # Be sure to read the max stream_ordering *before* checking if there are any outstanding # deltas, since there is otherwise a chance that we could miss updates which arrive # after we check the deltas. room_max_stream_ordering = self.store.get_room_max_stream_ordering( ) if self.pos == room_max_stream_ordering: break logger.debug("Processing room stats %s->%s", self.pos, room_max_stream_ordering) max_pos, deltas = await self.store.get_current_state_deltas( self.pos, room_max_stream_ordering) if deltas: logger.debug("Handling %d state deltas", len(deltas)) room_deltas, user_deltas = await self._handle_deltas(deltas) else: room_deltas = {} user_deltas = {} # Then count deltas for total_events and total_event_bytes. ( room_count, user_count, ) = await self.store.get_changes_room_total_events_and_bytes( self.pos, max_pos) for room_id, fields in room_count.items(): room_deltas.setdefault(room_id, {}).update(fields) for user_id, fields in user_count.items(): user_deltas.setdefault(user_id, {}).update(fields) logger.debug("room_deltas: %s", room_deltas) logger.debug("user_deltas: %s", user_deltas) # Always call this so that we update the stats position. await self.store.bulk_update_stats_delta( self.clock.time_msec(), updates={ "room": room_deltas, "user": user_deltas }, stream_id=max_pos, ) logger.debug("Handled room stats to %s -> %s", self.pos, max_pos) event_processing_positions.labels("stats").set(max_pos) self.pos = max_pos
async def _unsafe_process(self) -> None: # If self.pos is None then means we haven't fetched it from DB if self.pos is None: self.pos = await self.store.get_stats_positions() room_max_stream_ordering = self.store.get_room_max_stream_ordering( ) if self.pos > room_max_stream_ordering: # apparently, we've processed more events than exist in the database! # this can happen if events are removed with history purge or similar. logger.warning( "Event stream ordering appears to have gone backwards (%i -> %i): " "rewinding stats processor", self.pos, room_max_stream_ordering, ) self.pos = room_max_stream_ordering # Loop round handling deltas until we're up to date while True: # Be sure to read the max stream_ordering *before* checking if there are any outstanding # deltas, since there is otherwise a chance that we could miss updates which arrive # after we check the deltas. room_max_stream_ordering = self.store.get_room_max_stream_ordering( ) if self.pos == room_max_stream_ordering: break logger.debug("Processing room stats %s->%s", self.pos, room_max_stream_ordering) max_pos, deltas = await self.store.get_current_state_deltas( self.pos, room_max_stream_ordering) if deltas: logger.debug("Handling %d state deltas", len(deltas)) room_deltas, user_deltas = await self._handle_deltas(deltas) else: room_deltas = {} user_deltas = {} logger.debug("room_deltas: %s", room_deltas) logger.debug("user_deltas: %s", user_deltas) # Always call this so that we update the stats position. await self.store.bulk_update_stats_delta( self.clock.time_msec(), updates={ "room": room_deltas, "user": user_deltas }, stream_id=max_pos, ) logger.debug("Handled room stats to %s -> %s", self.pos, max_pos) event_processing_positions.labels("stats").set(max_pos) self.pos = max_pos
def _unsafe_process(self): # If self.pos is None then means we haven't fetched it from DB if self.pos is None: self.pos = yield self.store.get_stats_stream_pos() # If still None then the initial background update hasn't happened yet if self.pos is None: defer.returnValue(None) # Loop round handling deltas until we're up to date while True: with Measure(self.clock, "stats_delta"): deltas = yield self.store.get_current_state_deltas(self.pos) if not deltas: return logger.info("Handling %d state deltas", len(deltas)) yield self._handle_deltas(deltas) self.pos = deltas[-1]["stream_id"] yield self.store.update_stats_stream_pos(self.pos) event_processing_positions.labels("stats").set(self.pos)