async def paginate_room_events( self, room_id: str, from_key: str, to_key: Optional[str] = None, direction: str = "b", limit: int = -1, event_filter: Optional[Filter] = None, ) -> Tuple[List[EventBase], str]: """Returns list of events before or after a given token. Args: room_id from_key: The token used to stream from to_key: A token which if given limits the results to only those before direction: Either 'b' or 'f' to indicate whether we are paginating forwards or backwards from `from_key`. limit: The maximum number of events to return. event_filter: If provided filters the events to those that match the filter. Returns: The results as a list of events and a token that points to the end of the result set. If no events are returned then the end of the stream has been reached (i.e. there are no events between `from_key` and `to_key`). """ from_key = RoomStreamToken.parse(from_key) if to_key: to_key = RoomStreamToken.parse(to_key) rows, token = await self.db_pool.runInteraction( "paginate_room_events", self._paginate_room_events_txn, room_id, from_key, to_key, direction, limit, event_filter, ) events = await self.get_events_as_list( [r.event_id for r in rows], get_prev_content=True ) self._set_before_and_after(events, rows) return (events, token)
def paginate_room_events(self, room_id, from_key, to_key=None, direction='b', limit=-1, event_filter=None): """Returns list of events before or after a given token. Args: room_id (str) from_key (str): The token used to stream from to_key (str|None): A token which if given limits the results to only those before direction(char): Either 'b' or 'f' to indicate whether we are paginating forwards or backwards from `from_key`. limit (int): The maximum number of events to return. Zero or less means no limit. event_filter (Filter|None): If provided filters the events to those that match the filter. Returns: tuple[list[dict], str]: Returns the results as a list of dicts and a token that points to the end of the result set. The dicts have the keys "event_id", "topological_ordering" and "stream_orderign". """ from_key = RoomStreamToken.parse(from_key) if to_key: to_key = RoomStreamToken.parse(to_key) rows, token = yield self.runInteraction( "paginate_room_events", self._paginate_room_events_txn, room_id, from_key, to_key, direction, limit, event_filter, ) events = yield self._get_events([r.event_id for r in rows], get_prev_content=True) self._set_before_and_after(events, rows) defer.returnValue((events, token))
def paginate_room_events(self, room_id, from_key, to_key=None, direction="b", limit=-1, event_filter=None): """Returns list of events before or after a given token. Args: room_id (str) from_key (str): The token used to stream from to_key (str|None): A token which if given limits the results to only those before direction(char): Either 'b' or 'f' to indicate whether we are paginating forwards or backwards from `from_key`. limit (int): The maximum number of events to return. event_filter (Filter|None): If provided filters the events to those that match the filter. Returns: tuple[list[FrozenEvent], str]: Returns the results as a list of events and a token that points to the end of the result set. If no events are returned then the end of the stream has been reached (i.e. there are no events between `from_key` and `to_key`). """ from_key = RoomStreamToken.parse(from_key) if to_key: to_key = RoomStreamToken.parse(to_key) rows, token = yield self.runInteraction( "paginate_room_events", self._paginate_room_events_txn, room_id, from_key, to_key, direction, limit, event_filter, ) events = yield self.get_events_as_list([r.event_id for r in rows], get_prev_content=True) self._set_before_and_after(events, rows) return (events, token)
async def get_recent_event_ids_for_room( self, room_id: str, limit: int, end_token: str ) -> Tuple[List[_EventDictReturn], str]: """Get the most recent events in the room in topological ordering. Args: room_id limit end_token: The stream token representing now. Returns: A list of _EventDictReturn and a token pointing to the start of the returned events. The events returned are in ascending order. """ # Allow a zero limit here, and no-op. if limit == 0: return [], end_token end_token = RoomStreamToken.parse(end_token) rows, token = await self.db_pool.runInteraction( "get_recent_event_ids_for_room", self._paginate_room_events_txn, room_id, from_token=end_token, limit=limit, ) # We want to return the results in ascending order. rows.reverse() return rows, token
def get_recent_event_ids_for_room(self, room_id, limit, end_token): """Get the most recent events in the room in topological ordering. Args: room_id (str) limit (int) end_token (str): The stream token representing now. Returns: Deferred[tuple[list[_EventDictReturn], str]]: Returns a list of _EventDictReturn and a token pointing to the start of the returned events. The events returned are in ascending order. """ # Allow a zero limit here, and no-op. if limit == 0: defer.returnValue(([], end_token)) end_token = RoomStreamToken.parse(end_token) rows, token = yield self.runInteraction( "get_recent_event_ids_for_room", self._paginate_room_events_txn, room_id, from_token=end_token, limit=limit, ) # We want to return the results in ascending order. rows.reverse() defer.returnValue((rows, token))
def get_messages(self, user_id=None, room_id=None, pagin_config=None, feedback=False, as_client_event=True): """Get messages in a room. Args: user_id (str): The user requesting messages. room_id (str): The room they want messages from. pagin_config (synapse.api.streams.PaginationConfig): The pagination config rules to apply, if any. feedback (bool): True to get compressed feedback with the messages as_client_event (bool): True to get events in client-server format. Returns: dict: Pagination API results """ yield self.auth.check_joined_room(room_id, user_id) data_source = self.hs.get_event_sources().sources["room"] if not pagin_config.from_token: pagin_config.from_token = ( yield self.hs.get_event_sources().get_current_token(direction='b')) room_token = RoomStreamToken.parse(pagin_config.from_token.room_key) if room_token.topological is None: raise SynapseError(400, "Invalid token") yield self.hs.get_handlers().federation_handler.maybe_backfill( room_id, room_token.topological) user = UserID.from_string(user_id) events, next_key = yield data_source.get_pagination_rows( user, pagin_config.get_source_config("room"), room_id) next_token = pagin_config.from_token.copy_and_replace( "room_key", next_key) time_now = self.clock.time_msec() chunk = { "chunk": [serialize_event(e, time_now, as_client_event) for e in events], "start": pagin_config.from_token.to_string(), "end": next_token.to_string(), } defer.returnValue(chunk)
def get_new_events( self, user, from_key, limit, room_ids, is_guest, ): # We just ignore the key for now. to_key = yield self.get_current_key() from_token = RoomStreamToken.parse(from_key) if from_token.topological: logger.warn("Stream has topological part!!!! %r", from_key) from_key = "s%s" % (from_token.stream,) app_service = self.store.get_app_service_by_user_id( user.to_string() ) if app_service: events, end_key = yield self.store.get_appservice_room_stream( service=app_service, from_key=from_key, to_key=to_key, limit=limit, ) else: room_events = yield self.store.get_membership_changes_for_user( user.to_string(), from_key, to_key ) room_to_events = yield self.store.get_room_events_stream_for_rooms( room_ids=room_ids, from_key=from_key, to_key=to_key, limit=limit or 10, order='ASC', ) events = list(room_events) events.extend(e for evs, _ in room_to_events.values() for e in evs) events.sort(key=lambda e: e.internal_metadata.order) if limit: events[:] = events[:limit] if events: end_key = events[-1].internal_metadata.after else: end_key = to_key defer.returnValue((events, end_key))
def get_new_events( self, user, from_key, limit, room_ids, is_guest, ): # We just ignore the key for now. to_key = yield self.get_current_key() from_token = RoomStreamToken.parse(from_key) if from_token.topological: logger.warn("Stream has topological part!!!! %r", from_key) from_key = "s%s" % (from_token.stream,) app_service = yield self.store.get_app_service_by_user_id( user.to_string() ) if app_service: events, end_key = yield self.store.get_appservice_room_stream( service=app_service, from_key=from_key, to_key=to_key, limit=limit, ) else: room_events = yield self.store.get_membership_changes_for_user( user.to_string(), from_key, to_key ) room_to_events = yield self.store.get_room_events_stream_for_rooms( room_ids=room_ids, from_key=from_key, to_key=to_key, limit=limit or 10, order='ASC', ) events = list(room_events) events.extend(e for evs, _ in room_to_events.values() for e in evs) events.sort(key=lambda e: e.internal_metadata.order) if limit: events[:] = events[:limit] if events: end_key = events[-1].internal_metadata.after else: end_key = to_key defer.returnValue((events, end_key))
def get_messages(self, user_id=None, room_id=None, pagin_config=None, feedback=False, as_client_event=True): """Get messages in a room. Args: user_id (str): The user requesting messages. room_id (str): The room they want messages from. pagin_config (synapse.api.streams.PaginationConfig): The pagination config rules to apply, if any. feedback (bool): True to get compressed feedback with the messages as_client_event (bool): True to get events in client-server format. Returns: dict: Pagination API results """ yield self.auth.check_joined_room(room_id, user_id) data_source = self.hs.get_event_sources().sources["room"] if not pagin_config.from_token: pagin_config.from_token = ( yield self.hs.get_event_sources().get_current_token( direction='b' ) ) room_token = RoomStreamToken.parse(pagin_config.from_token.room_key) if room_token.topological is None: raise SynapseError(400, "Invalid token") yield self.hs.get_handlers().federation_handler.maybe_backfill( room_id, room_token.topological ) user = UserID.from_string(user_id) events, next_key = yield data_source.get_pagination_rows( user, pagin_config.get_source_config("room"), room_id ) next_token = pagin_config.from_token.copy_and_replace( "room_key", next_key ) time_now = self.clock.time_msec() chunk = { "chunk": [ serialize_event(e, time_now, as_client_event) for e in events ], "start": pagin_config.from_token.to_string(), "end": next_token.to_string(), } defer.returnValue(chunk)
def get_new_events( self, user, from_key, limit, room_ids, is_guest, explicit_room_id=None, ): # We just ignore the key for now. to_key = yield self.get_current_key() from_token = RoomStreamToken.parse(from_key) if from_token.topological: logger.warn("Stream has topological part!!!! %r", from_key) from_key = "s%s" % (from_token.stream,) app_service = self.store.get_app_service_by_user_id( user.to_string() ) if app_service: # We no longer support AS users using /sync directly. # See https://github.com/matrix-org/matrix-doc/issues/1144 raise NotImplementedError() else: room_events = yield self.store.get_membership_changes_for_user( user.to_string(), from_key, to_key ) room_to_events = yield self.store.get_room_events_stream_for_rooms( room_ids=room_ids, from_key=from_key, to_key=to_key, limit=limit or 10, order='ASC', ) events = list(room_events) events.extend(e for evs, _ in room_to_events.values() for e in evs) events.sort(key=lambda e: e.internal_metadata.order) if limit: events[:] = events[:limit] if events: end_key = events[-1].internal_metadata.after else: end_key = to_key defer.returnValue((events, end_key))
def paginate_room_events(self, room_id, from_key, to_key=None, direction='b', limit=-1, event_filter=None): """Returns list of events before or after a given token. Args: room_id (str) from_key (str): The token used to stream from to_key (str|None): A token which if given limits the results to only those before direction(char): Either 'b' or 'f' to indicate whether we are paginating forwards or backwards from `from_key`. limit (int): The maximum number of events to return. Zero or less means no limit. event_filter (Filter|None): If provided filters the events to those that match the filter. Returns: tuple[list[dict], str]: Returns the results as a list of dicts and a token that points to the end of the result set. The dicts have the keys "event_id", "topological_ordering" and "stream_orderign". """ from_key = RoomStreamToken.parse(from_key) if to_key: to_key = RoomStreamToken.parse(to_key) rows, token = yield self.runInteraction( "paginate_room_events", self._paginate_room_events_txn, room_id, from_key, to_key, direction, limit, event_filter, ) events = yield self._get_events( [r.event_id for r in rows], get_prev_content=True ) self._set_before_and_after(events, rows) defer.returnValue((events, token))
def get_messages( self, requester, room_id=None, pagin_config=None, as_client_event=True, event_filter=None, ): """Get messages in a room. Args: requester (Requester): The user requesting messages. room_id (str): The room they want messages from. pagin_config (synapse.api.streams.PaginationConfig): The pagination config rules to apply, if any. as_client_event (bool): True to get events in client-server format. event_filter (Filter): Filter to apply to results or None Returns: dict: Pagination API results """ user_id = requester.user.to_string() if pagin_config.from_token: room_token = pagin_config.from_token.room_key else: pagin_config.from_token = ( yield self.hs.get_event_sources().get_current_token_for_pagination() ) room_token = pagin_config.from_token.room_key room_token = RoomStreamToken.parse(room_token) pagin_config.from_token = pagin_config.from_token.copy_and_replace( "room_key", str(room_token) ) source_config = pagin_config.get_source_config("room") with (yield self.pagination_lock.read(room_id)): ( membership, member_event_id, ) = yield self.auth.check_in_room_or_world_readable(room_id, user_id) if source_config.direction == "b": # if we're going backwards, we might need to backfill. This # requires that we have a topo token. if room_token.topological: max_topo = room_token.topological else: max_topo = yield self.store.get_max_topological_token( room_id, room_token.stream ) if membership == Membership.LEAVE: # If they have left the room then clamp the token to be before # they left the room, to save the effort of loading from the # database. leave_token = yield self.store.get_topological_token_for_event( member_event_id ) leave_token = RoomStreamToken.parse(leave_token) if leave_token.topological < max_topo: source_config.from_key = str(leave_token) yield self.hs.get_handlers().federation_handler.maybe_backfill( room_id, max_topo ) events, next_key = yield self.store.paginate_room_events( room_id=room_id, from_key=source_config.from_key, to_key=source_config.to_key, direction=source_config.direction, limit=source_config.limit, event_filter=event_filter, ) next_token = pagin_config.from_token.copy_and_replace("room_key", next_key) if events: if event_filter: events = event_filter.filter(events) events = yield filter_events_for_client( self.storage, user_id, events, is_peeking=(member_event_id is None) ) if not events: return { "chunk": [], "start": pagin_config.from_token.to_string(), "end": next_token.to_string(), } state = None if event_filter and event_filter.lazy_load_members() and len(events) > 0: # TODO: remove redundant members # FIXME: we also care about invite targets etc. state_filter = StateFilter.from_types( (EventTypes.Member, event.sender) for event in events ) state_ids = yield self.state_store.get_state_ids_for_event( events[0].event_id, state_filter=state_filter ) if state_ids: state = yield self.store.get_events(list(state_ids.values())) state = state.values() time_now = self.clock.time_msec() chunk = { "chunk": ( yield self._event_serializer.serialize_events( events, time_now, as_client_event=as_client_event ) ), "start": pagin_config.from_token.to_string(), "end": next_token.to_string(), } if state: chunk["state"] = yield self._event_serializer.serialize_events( state, time_now, as_client_event=as_client_event ) return chunk
def paginate_room_events(self, room_id, from_key, to_key=None, direction='b', limit=-1, event_filter=None): # Tokens really represent positions between elements, but we use # the convention of pointing to the event before the gap. Hence # we have a bit of asymmetry when it comes to equalities. args = [False, room_id] if direction == 'b': order = "DESC" bounds = upper_bound( RoomStreamToken.parse(from_key), self.database_engine ) if to_key: bounds = "%s AND %s" % (bounds, lower_bound( RoomStreamToken.parse(to_key), self.database_engine )) else: order = "ASC" bounds = lower_bound( RoomStreamToken.parse(from_key), self.database_engine ) if to_key: bounds = "%s AND %s" % (bounds, upper_bound( RoomStreamToken.parse(to_key), self.database_engine )) filter_clause, filter_args = filter_to_clause(event_filter) if filter_clause: bounds += " AND " + filter_clause args.extend(filter_args) if int(limit) > 0: args.append(int(limit)) limit_str = " LIMIT ?" else: limit_str = "" sql = ( "SELECT * FROM events" " WHERE outlier = ? AND room_id = ? AND %(bounds)s" " ORDER BY topological_ordering %(order)s," " stream_ordering %(order)s %(limit)s" ) % { "bounds": bounds, "order": order, "limit": limit_str } def f(txn): txn.execute(sql, args) rows = self.cursor_to_dict(txn) if rows: topo = rows[-1]["topological_ordering"] toke = rows[-1]["stream_ordering"] if direction == 'b': # Tokens are positions between events. # This token points *after* the last event in the chunk. # We need it to point to the event before it in the chunk # when we are going backwards so we subtract one from the # stream part. toke -= 1 next_token = str(RoomStreamToken(topo, toke)) else: # TODO (erikj): We should work out what to do here instead. next_token = to_key if to_key else from_key return rows, next_token, rows, token = yield self.runInteraction("paginate_room_events", f) events = yield self._get_events( [r["event_id"] for r in rows], get_prev_content=True ) self._set_before_and_after(events, rows) defer.returnValue((events, token))
def paginate_room_events(self, room_id, from_key, to_key=None, direction='b', limit=-1, event_filter=None): # Tokens really represent positions between elements, but we use # the convention of pointing to the event before the gap. Hence # we have a bit of asymmetry when it comes to equalities. args = [False, room_id] if direction == 'b': order = "DESC" bounds = upper_bound(RoomStreamToken.parse(from_key), self.database_engine) if to_key: bounds = "%s AND %s" % (bounds, lower_bound( RoomStreamToken.parse(to_key), self.database_engine)) else: order = "ASC" bounds = lower_bound(RoomStreamToken.parse(from_key), self.database_engine) if to_key: bounds = "%s AND %s" % (bounds, upper_bound( RoomStreamToken.parse(to_key), self.database_engine)) filter_clause, filter_args = filter_to_clause(event_filter) if filter_clause: bounds += " AND " + filter_clause args.extend(filter_args) if int(limit) > 0: args.append(int(limit)) limit_str = " LIMIT ?" else: limit_str = "" sql = ("SELECT * FROM events" " WHERE outlier = ? AND room_id = ? AND %(bounds)s" " ORDER BY topological_ordering %(order)s," " stream_ordering %(order)s %(limit)s") % { "bounds": bounds, "order": order, "limit": limit_str } def f(txn): txn.execute(sql, args) rows = self.cursor_to_dict(txn) if rows: topo = rows[-1]["topological_ordering"] toke = rows[-1]["stream_ordering"] if direction == 'b': # Tokens are positions between events. # This token points *after* the last event in the chunk. # We need it to point to the event before it in the chunk # when we are going backwards so we subtract one from the # stream part. toke -= 1 next_token = str(RoomStreamToken(topo, toke)) else: # TODO (erikj): We should work out what to do here instead. next_token = to_key if to_key else from_key return rows, next_token, rows, token = yield self.runInteraction("paginate_room_events", f) events = yield self._get_events([r["event_id"] for r in rows], get_prev_content=True) self._set_before_and_after(events, rows) defer.returnValue((events, token))
def get_messages(self, user_id=None, room_id=None, pagin_config=None, as_client_event=True, is_guest=False): """Get messages in a room. Args: user_id (str): The user requesting messages. room_id (str): The room they want messages from. pagin_config (synapse.api.streams.PaginationConfig): The pagination config rules to apply, if any. as_client_event (bool): True to get events in client-server format. is_guest (bool): Whether the requesting user is a guest (as opposed to a fully registered user). Returns: dict: Pagination API results """ data_source = self.hs.get_event_sources().sources["room"] if pagin_config.from_token: room_token = pagin_config.from_token.room_key else: pagin_config.from_token = ( yield self.hs.get_event_sources().get_current_token( direction='b' ) ) room_token = pagin_config.from_token.room_key room_token = RoomStreamToken.parse(room_token) if room_token.topological is None: raise SynapseError(400, "Invalid token") pagin_config.from_token = pagin_config.from_token.copy_and_replace( "room_key", str(room_token) ) source_config = pagin_config.get_source_config("room") if not is_guest: member_event = yield self.auth.check_user_was_in_room(room_id, user_id) if member_event.membership == Membership.LEAVE: # If they have left the room then clamp the token to be before # they left the room. # If they're a guest, we'll just 403 them if they're asking for # events they can't see. leave_token = yield self.store.get_topological_token_for_event( member_event.event_id ) leave_token = RoomStreamToken.parse(leave_token) if leave_token.topological < room_token.topological: source_config.from_key = str(leave_token) if source_config.direction == "f": if source_config.to_key is None: source_config.to_key = str(leave_token) else: to_token = RoomStreamToken.parse(source_config.to_key) if leave_token.topological < to_token.topological: source_config.to_key = str(leave_token) yield self.hs.get_handlers().federation_handler.maybe_backfill( room_id, room_token.topological ) user = UserID.from_string(user_id) events, next_key = yield data_source.get_pagination_rows( user, source_config, room_id ) next_token = pagin_config.from_token.copy_and_replace( "room_key", next_key ) if not events: defer.returnValue({ "chunk": [], "start": pagin_config.from_token.to_string(), "end": next_token.to_string(), }) events = yield self._filter_events_for_client(user_id, events, is_guest=is_guest) time_now = self.clock.time_msec() chunk = { "chunk": [ serialize_event(e, time_now, as_client_event) for e in events ], "start": pagin_config.from_token.to_string(), "end": next_token.to_string(), } defer.returnValue(chunk)
def get_messages(self, requester, room_id=None, pagin_config=None, as_client_event=True, event_filter=None): """Get messages in a room. Args: requester (Requester): The user requesting messages. room_id (str): The room they want messages from. pagin_config (synapse.api.streams.PaginationConfig): The pagination config rules to apply, if any. as_client_event (bool): True to get events in client-server format. event_filter (Filter): Filter to apply to results or None Returns: dict: Pagination API results """ user_id = requester.user.to_string() if pagin_config.from_token: room_token = pagin_config.from_token.room_key else: pagin_config.from_token = ( yield self.hs.get_event_sources().get_current_token_for_room( room_id=room_id ) ) room_token = pagin_config.from_token.room_key room_token = RoomStreamToken.parse(room_token) pagin_config.from_token = pagin_config.from_token.copy_and_replace( "room_key", str(room_token) ) source_config = pagin_config.get_source_config("room") with (yield self.pagination_lock.read(room_id)): membership, member_event_id = yield self.auth.check_in_room_or_world_readable( room_id, user_id ) if source_config.direction == 'b': # if we're going backwards, we might need to backfill. This # requires that we have a topo token. if room_token.topological: max_topo = room_token.topological else: max_topo = yield self.store.get_max_topological_token( room_id, room_token.stream ) if membership == Membership.LEAVE: # If they have left the room then clamp the token to be before # they left the room, to save the effort of loading from the # database. leave_token = yield self.store.get_topological_token_for_event( member_event_id ) leave_token = RoomStreamToken.parse(leave_token) if leave_token.topological < max_topo: source_config.from_key = str(leave_token) yield self.hs.get_handlers().federation_handler.maybe_backfill( room_id, max_topo ) events, next_key = yield self.store.paginate_room_events( room_id=room_id, from_key=source_config.from_key, to_key=source_config.to_key, direction=source_config.direction, limit=source_config.limit, event_filter=event_filter, ) next_token = pagin_config.from_token.copy_and_replace( "room_key", next_key ) if events: if event_filter: events = event_filter.filter(events) events = yield filter_events_for_client( self.store, user_id, events, is_peeking=(member_event_id is None), ) if not events: defer.returnValue({ "chunk": [], "start": pagin_config.from_token.to_string(), "end": next_token.to_string(), }) state = None if event_filter and event_filter.lazy_load_members(): # TODO: remove redundant members # FIXME: we also care about invite targets etc. state_filter = StateFilter.from_types( (EventTypes.Member, event.sender) for event in events ) state_ids = yield self.store.get_state_ids_for_event( events[0].event_id, state_filter=state_filter, ) if state_ids: state = yield self.store.get_events(list(state_ids.values())) state = state.values() time_now = self.clock.time_msec() chunk = { "chunk": [ serialize_event(e, time_now, as_client_event) for e in events ], "start": pagin_config.from_token.to_string(), "end": next_token.to_string(), } if state: chunk["state"] = [ serialize_event(e, time_now, as_client_event) for e in state ] defer.returnValue(chunk)
def _purge_history_txn(self, txn, room_id, token_str, delete_local_events): token = RoomStreamToken.parse(token_str) # Tables that should be pruned: # event_auth # event_backward_extremities # event_edges # event_forward_extremities # event_json # event_push_actions # event_reference_hashes # event_relations # event_search # event_to_state_groups # events # rejections # room_depth # state_groups # state_groups_state # we will build a temporary table listing the events so that we don't # have to keep shovelling the list back and forth across the # connection. Annoyingly the python sqlite driver commits the # transaction on CREATE, so let's do this first. # # furthermore, we might already have the table from a previous (failed) # purge attempt, so let's drop the table first. txn.execute("DROP TABLE IF EXISTS events_to_purge") txn.execute("CREATE TEMPORARY TABLE events_to_purge (" " event_id TEXT NOT NULL," " should_delete BOOLEAN NOT NULL" ")") # First ensure that we're not about to delete all the forward extremeties txn.execute( "SELECT e.event_id, e.depth FROM events as e " "INNER JOIN event_forward_extremities as f " "ON e.event_id = f.event_id " "AND e.room_id = f.room_id " "WHERE f.room_id = ?", (room_id, ), ) rows = txn.fetchall() max_depth = max(row[1] for row in rows) if max_depth < token.topological: # We need to ensure we don't delete all the events from the database # otherwise we wouldn't be able to send any events (due to not # having any backwards extremeties) raise SynapseError( 400, "topological_ordering is greater than forward extremeties") logger.info("[purge] looking for events to delete") should_delete_expr = "state_key IS NULL" should_delete_params = () # type: Tuple[Any, ...] if not delete_local_events: should_delete_expr += " AND event_id NOT LIKE ?" # We include the parameter twice since we use the expression twice should_delete_params += ("%:" + self.hs.hostname, "%:" + self.hs.hostname) should_delete_params += (room_id, token.topological) # Note that we insert events that are outliers and aren't going to be # deleted, as nothing will happen to them. txn.execute( "INSERT INTO events_to_purge" " SELECT event_id, %s" " FROM events AS e LEFT JOIN state_events USING (event_id)" " WHERE (NOT outlier OR (%s)) AND e.room_id = ? AND topological_ordering < ?" % (should_delete_expr, should_delete_expr), should_delete_params, ) # We create the indices *after* insertion as that's a lot faster. # create an index on should_delete because later we'll be looking for # the should_delete / shouldn't_delete subsets txn.execute("CREATE INDEX events_to_purge_should_delete" " ON events_to_purge(should_delete)") # We do joins against events_to_purge for e.g. calculating state # groups to purge, etc., so lets make an index. txn.execute( "CREATE INDEX events_to_purge_id ON events_to_purge(event_id)") txn.execute("SELECT event_id, should_delete FROM events_to_purge") event_rows = txn.fetchall() logger.info( "[purge] found %i events before cutoff, of which %i can be deleted", len(event_rows), sum(1 for e in event_rows if e[1]), ) logger.info("[purge] Finding new backward extremities") # We calculate the new entries for the backward extremeties by finding # events to be purged that are pointed to by events we're not going to # purge. txn.execute( "SELECT DISTINCT e.event_id FROM events_to_purge AS e" " INNER JOIN event_edges AS ed ON e.event_id = ed.prev_event_id" " LEFT JOIN events_to_purge AS ep2 ON ed.event_id = ep2.event_id" " WHERE ep2.event_id IS NULL") new_backwards_extrems = txn.fetchall() logger.info("[purge] replacing backward extremities: %r", new_backwards_extrems) txn.execute("DELETE FROM event_backward_extremities WHERE room_id = ?", (room_id, )) # Update backward extremeties txn.executemany( "INSERT INTO event_backward_extremities (room_id, event_id)" " VALUES (?, ?)", [(room_id, event_id) for event_id, in new_backwards_extrems], ) logger.info( "[purge] finding state groups referenced by deleted events") # Get all state groups that are referenced by events that are to be # deleted. txn.execute(""" SELECT DISTINCT state_group FROM events_to_purge INNER JOIN event_to_state_groups USING (event_id) """) referenced_state_groups = {sg for sg, in txn} logger.info("[purge] found %i referenced state groups", len(referenced_state_groups)) logger.info("[purge] removing events from event_to_state_groups") txn.execute("DELETE FROM event_to_state_groups " "WHERE event_id IN (SELECT event_id from events_to_purge)") for event_id, _ in event_rows: txn.call_after(self._get_state_group_for_event.invalidate, (event_id, )) # Delete all remote non-state events for table in ( "events", "event_json", "event_auth", "event_edges", "event_forward_extremities", "event_reference_hashes", "event_relations", "event_search", "rejections", ): logger.info("[purge] removing events from %s", table) txn.execute( "DELETE FROM %s WHERE event_id IN (" " SELECT event_id FROM events_to_purge WHERE should_delete" ")" % (table, )) # event_push_actions lacks an index on event_id, and has one on # (room_id, event_id) instead. for table in ("event_push_actions", ): logger.info("[purge] removing events from %s", table) txn.execute( "DELETE FROM %s WHERE room_id = ? AND event_id IN (" " SELECT event_id FROM events_to_purge WHERE should_delete" ")" % (table, ), (room_id, ), ) # Mark all state and own events as outliers logger.info("[purge] marking remaining events as outliers") txn.execute( "UPDATE events SET outlier = ?" " WHERE event_id IN (" " SELECT event_id FROM events_to_purge " " WHERE NOT should_delete" ")", (True, ), ) # synapse tries to take out an exclusive lock on room_depth whenever it # persists events (because upsert), and once we run this update, we # will block that for the rest of our transaction. # # So, let's stick it at the end so that we don't block event # persistence. # # We do this by calculating the minimum depth of the backwards # extremities. However, the events in event_backward_extremities # are ones we don't have yet so we need to look at the events that # point to it via event_edges table. txn.execute( """ SELECT COALESCE(MIN(depth), 0) FROM event_backward_extremities AS eb INNER JOIN event_edges AS eg ON eg.prev_event_id = eb.event_id INNER JOIN events AS e ON e.event_id = eg.event_id WHERE eb.room_id = ? """, (room_id, ), ) (min_depth, ) = txn.fetchone() logger.info("[purge] updating room_depth to %d", min_depth) txn.execute( "UPDATE room_depth SET min_depth = ? WHERE room_id = ?", (min_depth, room_id), ) # finally, drop the temp table. this will commit the txn in sqlite, # so make sure to keep this actually last. txn.execute("DROP TABLE events_to_purge") logger.info("[purge] done") return referenced_state_groups
def get_messages(self, requester, room_id=None, pagin_config=None, as_client_event=True): """Get messages in a room. Args: requester (Requester): The user requesting messages. room_id (str): The room they want messages from. pagin_config (synapse.api.streams.PaginationConfig): The pagination config rules to apply, if any. as_client_event (bool): True to get events in client-server format. Returns: dict: Pagination API results """ user_id = requester.user.to_string() data_source = self.hs.get_event_sources().sources["room"] if pagin_config.from_token: room_token = pagin_config.from_token.room_key else: pagin_config.from_token = ( yield self.hs.get_event_sources().get_current_token( direction='b' ) ) room_token = pagin_config.from_token.room_key room_token = RoomStreamToken.parse(room_token) pagin_config.from_token = pagin_config.from_token.copy_and_replace( "room_key", str(room_token) ) source_config = pagin_config.get_source_config("room") membership, member_event_id = yield self._check_in_room_or_world_readable( room_id, user_id ) if source_config.direction == 'b': # if we're going backwards, we might need to backfill. This # requires that we have a topo token. if room_token.topological: max_topo = room_token.topological else: max_topo = yield self.store.get_max_topological_token_for_stream_and_room( room_id, room_token.stream ) if membership == Membership.LEAVE: # If they have left the room then clamp the token to be before # they left the room, to save the effort of loading from the # database. leave_token = yield self.store.get_topological_token_for_event( member_event_id ) leave_token = RoomStreamToken.parse(leave_token) if leave_token.topological < max_topo: source_config.from_key = str(leave_token) yield self.hs.get_handlers().federation_handler.maybe_backfill( room_id, max_topo ) events, next_key = yield data_source.get_pagination_rows( requester.user, source_config, room_id ) next_token = pagin_config.from_token.copy_and_replace( "room_key", next_key ) if not events: defer.returnValue({ "chunk": [], "start": pagin_config.from_token.to_string(), "end": next_token.to_string(), }) events = yield self._filter_events_for_client( user_id, events, is_peeking=(member_event_id is None), ) time_now = self.clock.time_msec() chunk = { "chunk": [ serialize_event(e, time_now, as_client_event) for e in events ], "start": pagin_config.from_token.to_string(), "end": next_token.to_string(), } defer.returnValue(chunk)
def get_messages(self, requester, room_id=None, pagin_config=None, as_client_event=True, event_filter=None): """Get messages in a room. Args: requester (Requester): The user requesting messages. room_id (str): The room they want messages from. pagin_config (synapse.api.streams.PaginationConfig): The pagination config rules to apply, if any. as_client_event (bool): True to get events in client-server format. event_filter (Filter): Filter to apply to results or None Returns: dict: Pagination API results """ user_id = requester.user.to_string() if pagin_config.from_token: room_token = pagin_config.from_token.room_key else: pagin_config.from_token = ( yield self.hs.get_event_sources().get_current_token_for_room( room_id=room_id)) room_token = pagin_config.from_token.room_key room_token = RoomStreamToken.parse(room_token) pagin_config.from_token = pagin_config.from_token.copy_and_replace( "room_key", str(room_token)) source_config = pagin_config.get_source_config("room") with (yield self.pagination_lock.read(room_id)): membership, member_event_id = yield self._check_in_room_or_world_readable( room_id, user_id) if source_config.direction == 'b': # if we're going backwards, we might need to backfill. This # requires that we have a topo token. if room_token.topological: max_topo = room_token.topological else: max_topo = yield self.store.get_max_topological_token( room_id, room_token.stream) if membership == Membership.LEAVE: # If they have left the room then clamp the token to be before # they left the room, to save the effort of loading from the # database. leave_token = yield self.store.get_topological_token_for_event( member_event_id) leave_token = RoomStreamToken.parse(leave_token) if leave_token.topological < max_topo: source_config.from_key = str(leave_token) yield self.hs.get_handlers().federation_handler.maybe_backfill( room_id, max_topo) events, next_key = yield self.store.paginate_room_events( room_id=room_id, from_key=source_config.from_key, to_key=source_config.to_key, direction=source_config.direction, limit=source_config.limit, event_filter=event_filter, ) next_token = pagin_config.from_token.copy_and_replace( "room_key", next_key) if not events: defer.returnValue({ "chunk": [], "start": pagin_config.from_token.to_string(), "end": next_token.to_string(), }) if event_filter: events = event_filter.filter(events) events = yield filter_events_for_client( self.store, user_id, events, is_peeking=(member_event_id is None), ) time_now = self.clock.time_msec() chunk = { "chunk": [serialize_event(e, time_now, as_client_event) for e in events], "start": pagin_config.from_token.to_string(), "end": next_token.to_string(), } defer.returnValue(chunk)
async def get_messages( self, requester: Requester, room_id: str, pagin_config: PaginationConfig, as_client_event: bool = True, event_filter: Optional[Filter] = None, ) -> Dict[str, Any]: """Get messages in a room. Args: requester: The user requesting messages. room_id: The room they want messages from. pagin_config: The pagination config rules to apply, if any. as_client_event: True to get events in client-server format. event_filter: Filter to apply to results or None Returns: Pagination API results """ user_id = requester.user.to_string() if pagin_config.from_token: room_token = pagin_config.from_token.room_key else: pagin_config.from_token = ( self.hs.get_event_sources().get_current_token_for_pagination()) room_token = pagin_config.from_token.room_key room_token = RoomStreamToken.parse(room_token) pagin_config.from_token = pagin_config.from_token.copy_and_replace( "room_key", str(room_token)) source_config = pagin_config.get_source_config("room") with await self.pagination_lock.read(room_id): ( membership, member_event_id, ) = await self.auth.check_user_in_room_or_world_readable( room_id, user_id, allow_departed_users=True) if source_config.direction == "b": # if we're going backwards, we might need to backfill. This # requires that we have a topo token. if room_token.topological: curr_topo = room_token.topological else: curr_topo = await self.store.get_current_topological_token( room_id, room_token.stream) if membership == Membership.LEAVE: # If they have left the room then clamp the token to be before # they left the room, to save the effort of loading from the # database. # This is only None if the room is world_readable, in which # case "JOIN" would have been returned. assert member_event_id leave_token = await self.store.get_topological_token_for_event( member_event_id) if RoomStreamToken.parse( leave_token).topological < curr_topo: source_config.from_key = str(leave_token) await self.hs.get_handlers().federation_handler.maybe_backfill( room_id, curr_topo, limit=source_config.limit, ) events, next_key = await self.store.paginate_room_events( room_id=room_id, from_key=source_config.from_key, to_key=source_config.to_key, direction=source_config.direction, limit=source_config.limit, event_filter=event_filter, ) next_token = pagin_config.from_token.copy_and_replace( "room_key", next_key) if events: if event_filter: events = event_filter.filter(events) events = await filter_events_for_client( self.storage, user_id, events, is_peeking=(member_event_id is None)) if not events: return { "chunk": [], "start": pagin_config.from_token.to_string(), "end": next_token.to_string(), } state = None if event_filter and event_filter.lazy_load_members( ) and len(events) > 0: # TODO: remove redundant members # FIXME: we also care about invite targets etc. state_filter = StateFilter.from_types( (EventTypes.Member, event.sender) for event in events) state_ids = await self.state_store.get_state_ids_for_event( events[0].event_id, state_filter=state_filter) if state_ids: state_dict = await self.store.get_events( list(state_ids.values())) state = state_dict.values() time_now = self.clock.time_msec() chunk = { "chunk": (await self._event_serializer.serialize_events( events, time_now, as_client_event=as_client_event)), "start": pagin_config.from_token.to_string(), "end": next_token.to_string(), } if state: chunk["state"] = await self._event_serializer.serialize_events( state, time_now, as_client_event=as_client_event) return chunk