def get_recent_event_ids_for_room(self, room_id, limit, end_token, from_token=None): end_token = RoomStreamToken.parse_stream_token(end_token) if from_token is None: sql = ( "SELECT stream_ordering, topological_ordering, event_id" " FROM events" " WHERE room_id = ? AND stream_ordering <= ? AND outlier = ?" " ORDER BY topological_ordering DESC, stream_ordering DESC" " LIMIT ?" ) else: from_token = RoomStreamToken.parse_stream_token(from_token) sql = ( "SELECT stream_ordering, topological_ordering, event_id" " FROM events" " WHERE room_id = ? AND stream_ordering > ?" " AND stream_ordering <= ? AND outlier = ?" " ORDER BY topological_ordering DESC, stream_ordering DESC" " LIMIT ?" ) def get_recent_events_for_room_txn(txn): if from_token is None: txn.execute(sql, (room_id, end_token.stream, False, limit,)) else: txn.execute(sql, ( room_id, from_token.stream, end_token.stream, False, limit )) rows = self.cursor_to_dict(txn) rows.reverse() # As we selected with reverse ordering if rows: # Tokens are positions between events. # This token points *after* the last event in the chunk. # We need it to point to the event before it in the chunk # since we are going backwards so we subtract one from the # stream part. topo = rows[0]["topological_ordering"] toke = rows[0]["stream_ordering"] - 1 start_token = str(RoomStreamToken(topo, toke)) token = (start_token, str(end_token)) else: token = (str(end_token), str(end_token)) return rows, token return self.runInteraction( "get_recent_events_for_room", get_recent_events_for_room_txn )
def get_membership_changes_for_user(self, user_id, from_key, to_key): if from_key is not None: from_id = RoomStreamToken.parse_stream_token(from_key).stream else: from_id = None to_id = RoomStreamToken.parse_stream_token(to_key).stream if from_key == to_key: defer.returnValue([]) if from_id: has_changed = self._membership_stream_cache.has_entity_changed( user_id, int(from_id) ) if not has_changed: defer.returnValue([]) def f(txn): if from_id is not None: sql = ( "SELECT m.event_id, stream_ordering FROM events AS e," " room_memberships AS m" " WHERE e.event_id = m.event_id" " AND m.user_id = ?" " AND e.stream_ordering > ? AND e.stream_ordering <= ?" " ORDER BY e.stream_ordering ASC" ) txn.execute(sql, (user_id, from_id, to_id,)) else: sql = ( "SELECT m.event_id, stream_ordering FROM events AS e," " room_memberships AS m" " WHERE e.event_id = m.event_id" " AND m.user_id = ?" " AND stream_ordering <= ?" " ORDER BY stream_ordering ASC" ) txn.execute(sql, (user_id, to_id,)) rows = self.cursor_to_dict(txn) return rows rows = yield self.runInteraction("get_membership_changes_for_user", f) ret = yield self._get_events( [r["event_id"] for r in rows], get_prev_content=True ) self._set_before_and_after(ret, rows, topo_order=False) defer.returnValue(ret)
def get_membership_changes_for_user(self, user_id, from_key, to_key): if from_key is not None: from_id = RoomStreamToken.parse_stream_token(from_key).stream else: from_id = None to_id = RoomStreamToken.parse_stream_token(to_key).stream if from_key == to_key: defer.returnValue([]) if from_id: has_changed = self._membership_stream_cache.has_entity_changed( user_id, int(from_id)) if not has_changed: defer.returnValue([]) def f(txn): if from_id is not None: sql = ("SELECT m.event_id, stream_ordering FROM events AS e," " room_memberships AS m" " WHERE e.event_id = m.event_id" " AND m.user_id = ?" " AND e.stream_ordering > ? AND e.stream_ordering <= ?" " ORDER BY e.stream_ordering ASC") txn.execute(sql, ( user_id, from_id, to_id, )) else: sql = ("SELECT m.event_id, stream_ordering FROM events AS e," " room_memberships AS m" " WHERE e.event_id = m.event_id" " AND m.user_id = ?" " AND stream_ordering <= ?" " ORDER BY stream_ordering ASC") txn.execute(sql, ( user_id, to_id, )) rows = self.cursor_to_dict(txn) return rows rows = yield self.runInteraction("get_membership_changes_for_user", f) ret = yield self._get_events([r["event_id"] for r in rows], get_prev_content=True) self._set_before_and_after(ret, rows, topo_order=False) defer.returnValue(ret)
def get_room_events_stream_for_rooms(self, room_ids, from_key, to_key, limit=0, order='DESC'): from_id = RoomStreamToken.parse_stream_token(from_key).stream room_ids = yield self._events_stream_cache.get_entities_changed( room_ids, from_id) if not room_ids: defer.returnValue({}) results = {} room_ids = list(room_ids) for rm_ids in (room_ids[i:i + 20] for i in xrange(0, len(room_ids), 20)): res = yield preserve_context_over_deferred( defer.gatherResults([ preserve_fn(self.get_room_events_stream_for_room)( room_id, from_key, to_key, limit, order=order, ) for room_id in rm_ids ])) results.update(dict(zip(rm_ids, res))) defer.returnValue(results)
def get_room_events_stream_for_rooms(self, room_ids, from_key, to_key, limit=0, order='DESC'): from_id = RoomStreamToken.parse_stream_token(from_key).stream room_ids = yield self._events_stream_cache.get_entities_changed( room_ids, from_id) if not room_ids: defer.returnValue({}) results = {} room_ids = list(room_ids) for rm_ids in (room_ids[i:i + 20] for i in range(0, len(room_ids), 20)): res = yield make_deferred_yieldable( defer.gatherResults([ run_in_background( self.get_room_events_stream_for_room, room_id, from_key, to_key, limit, order=order, ) for room_id in rm_ids ], consumeErrors=True)) results.update(dict(zip(rm_ids, res))) defer.returnValue(results)
def _have_rooms_changed(self, sync_result_builder): """Returns whether there may be any new events that should be sent down the sync. Returns True if there are. """ user_id = sync_result_builder.sync_config.user.to_string() since_token = sync_result_builder.since_token now_token = sync_result_builder.now_token assert since_token # Get a list of membership change events that have happened. rooms_changed = yield self.store.get_membership_changes_for_user( user_id, since_token.room_key, now_token.room_key ) if rooms_changed: defer.returnValue(True) app_service = self.store.get_app_service_by_user_id(user_id) if app_service: rooms = yield self.store.get_app_service_rooms(app_service) joined_room_ids = set(r.room_id for r in rooms) else: joined_room_ids = yield self.store.get_rooms_for_user(user_id) stream_id = RoomStreamToken.parse_stream_token(since_token.room_key).stream for room_id in joined_room_ids: if self.store.has_room_changed_since(room_id, stream_id): defer.returnValue(True) defer.returnValue(False)
async def get_room_events_stream_for_rooms( self, room_ids: Iterable[str], from_key: str, to_key: str, limit: int = 0, order: str = "DESC", ) -> Dict[str, Tuple[List[EventBase], str]]: """Get new room events in stream ordering since `from_key`. Args: room_ids from_key: Token from which no events are returned before to_key: Token from which no events are returned after. (This is typically the current stream token) limit: Maximum number of events to return order: Either "DESC" or "ASC". Determines which events are returned when the result is limited. If "DESC" then the most recent `limit` events are returned, otherwise returns the oldest `limit` events. Returns: A map from room id to a tuple containing: - list of recent events in the room - stream ordering key for the start of the chunk of events returned. """ from_id = RoomStreamToken.parse_stream_token(from_key).stream room_ids = self._events_stream_cache.get_entities_changed(room_ids, from_id) if not room_ids: return {} results = {} room_ids = list(room_ids) for rm_ids in (room_ids[i : i + 20] for i in range(0, len(room_ids), 20)): res = await make_deferred_yieldable( defer.gatherResults( [ run_in_background( self.get_room_events_stream_for_room, room_id, from_key, to_key, limit, order=order, ) for room_id in rm_ids ], consumeErrors=True, ) ) results.update(dict(zip(rm_ids, res))) return results
def get_rooms_that_changed(self, room_ids, from_key): """Given a list of rooms and a token, return rooms where there may have been changes. Args: room_ids (list) from_key (str): The room_key portion of a StreamToken """ from_key = RoomStreamToken.parse_stream_token(from_key).stream return set( room_id for room_id in room_ids if self._events_stream_cache.has_entity_changed(room_id, from_key))
def get_rooms_that_changed(self, room_ids, from_key): """Given a list of rooms and a token, return rooms where there may have been changes. Args: room_ids (list) from_key (str): The room_key portion of a StreamToken """ from_key = RoomStreamToken.parse_stream_token(from_key).stream return set( room_id for room_id in room_ids if self._events_stream_cache.has_entity_changed(room_id, from_key) )
def get_user_ids_changed(self, user_id, from_token): """Get list of users that have had the devices updated, or have newly joined a room, that `user_id` may be interested in. Args: user_id (str) from_token (StreamToken) """ rooms = yield self.store.get_rooms_for_user(user_id) room_ids = set(r.room_id for r in rooms) # First we check if any devices have changed changed = yield self.store.get_user_whose_devices_changed( from_token.device_list_key) # Then work out if any users have since joined rooms_changed = self.store.get_rooms_that_changed( room_ids, from_token.room_key) possibly_changed = set(changed) for room_id in rooms_changed: # Fetch the current state at the time. stream_ordering = RoomStreamToken.parse_stream_token( from_token.room_key) try: event_ids = yield self.store.get_forward_extremeties_for_room( room_id, stream_ordering=stream_ordering) prev_state_ids = yield self.store.get_state_ids_for_events( event_ids) except: prev_state_ids = {} current_state_ids = yield self.state.get_current_state_ids(room_id) # If there has been any change in membership, include them in the # possibly changed list. We'll check if they are joined below, # and we're not toooo worried about spuriously adding users. for key, event_id in current_state_ids.iteritems(): etype, state_key = key if etype == EventTypes.Member: prev_event_id = prev_state_ids.get(key, None) if not prev_event_id or prev_event_id != event_id: possibly_changed.add(state_key) users_who_share_room = yield self.store.get_users_who_share_room_with_user( user_id) # Take the intersection of the users whose devices may have changed # and those that actually still share a room with the user defer.returnValue(users_who_share_room & possibly_changed)
def get_membership_changes_for_user(self, user_id, from_key, to_key): from_id = RoomStreamToken.parse_stream_token(from_key).stream to_id = RoomStreamToken.parse_stream_token(to_key).stream if from_key == to_key: defer.returnValue([]) if from_id: has_changed = self._membership_stream_cache.has_entity_changed( user_id, int(from_id) ) if not has_changed: defer.returnValue([]) def f(txn): sql = ( "SELECT m.event_id, stream_ordering FROM events AS e," " room_memberships AS m" " WHERE e.event_id = m.event_id" " AND m.user_id = ?" " AND e.stream_ordering > ? AND e.stream_ordering <= ?" " ORDER BY e.stream_ordering ASC" ) txn.execute(sql, (user_id, from_id, to_id)) rows = [_EventDictReturn(row[0], None, row[1]) for row in txn] return rows rows = yield self.runInteraction("get_membership_changes_for_user", f) ret = yield self.get_events_as_list( [r.event_id for r in rows], get_prev_content=True, ) self._set_before_and_after(ret, rows, topo_order=False) defer.returnValue(ret)
def get_rooms_that_changed(self, room_ids: Collection[str], from_key: str) -> Set[str]: """Given a list of rooms and a token, return rooms where there may have been changes. Args: room_ids from_key: The room_key portion of a StreamToken """ from_id = RoomStreamToken.parse_stream_token(from_key).stream return { room_id for room_id in room_ids if self._events_stream_cache.has_entity_changed(room_id, from_id) }
def get_room_events_stream_for_rooms(self, room_ids, from_key, to_key, limit=0, order='DESC'): """Get new room events in stream ordering since `from_key`. Args: room_id (str) from_key (str): Token from which no events are returned before to_key (str): Token from which no events are returned after. (This is typically the current stream token) limit (int): Maximum number of events to return order (str): Either "DESC" or "ASC". Determines which events are returned when the result is limited. If "DESC" then the most recent `limit` events are returned, otherwise returns the oldest `limit` events. Returns: Deferred[dict[str,tuple[list[FrozenEvent], str]]] A map from room id to a tuple containing: - list of recent events in the room - stream ordering key for the start of the chunk of events returned. """ from_id = RoomStreamToken.parse_stream_token(from_key).stream room_ids = yield self._events_stream_cache.get_entities_changed( room_ids, from_id ) if not room_ids: defer.returnValue({}) results = {} room_ids = list(room_ids) for rm_ids in (room_ids[i:i + 20] for i in range(0, len(room_ids), 20)): res = yield make_deferred_yieldable(defer.gatherResults([ run_in_background( self.get_room_events_stream_for_room, room_id, from_key, to_key, limit, order=order, ) for room_id in rm_ids ], consumeErrors=True)) results.update(dict(zip(rm_ids, res))) defer.returnValue(results)
def get_room_events_stream_for_rooms(self, room_ids, from_key, to_key, limit=0, order='DESC'): from_id = RoomStreamToken.parse_stream_token(from_key).stream room_ids = yield self._events_stream_cache.get_entities_changed( room_ids, from_id ) if not room_ids: defer.returnValue({}) results = {} room_ids = list(room_ids) for rm_ids in (room_ids[i:i + 20] for i in xrange(0, len(room_ids), 20)): res = yield defer.gatherResults([ preserve_fn(self.get_room_events_stream_for_room)( room_id, from_key, to_key, limit, order=order, ) for room_id in rm_ids ]) results.update(dict(zip(rm_ids, res))) defer.returnValue(results)
def get_room_events_stream_for_rooms(self, room_ids, from_key, to_key, limit=0, order='DESC'): from_id = RoomStreamToken.parse_stream_token(from_key).stream room_ids = yield self._events_stream_cache.get_entities_changed( room_ids, from_id ) if not room_ids: defer.returnValue({}) results = {} room_ids = list(room_ids) for rm_ids in (room_ids[i:i + 20] for i in range(0, len(room_ids), 20)): res = yield make_deferred_yieldable(defer.gatherResults([ run_in_background( self.get_room_events_stream_for_room, room_id, from_key, to_key, limit, order=order, ) for room_id in rm_ids ], consumeErrors=True)) results.update(dict(zip(rm_ids, res))) defer.returnValue(results)
def get_user_ids_changed(self, user_id, from_token): """Get list of users that have had the devices updated, or have newly joined a room, that `user_id` may be interested in. Args: user_id (str) from_token (StreamToken) """ now_token = yield self.hs.get_event_sources().get_current_token() room_ids = yield self.store.get_rooms_for_user(user_id) # First we check if any devices have changed changed = yield self.store.get_user_whose_devices_changed( from_token.device_list_key) # Then work out if any users have since joined rooms_changed = self.store.get_rooms_that_changed( room_ids, from_token.room_key) member_events = yield self.store.get_membership_changes_for_user( user_id, from_token.room_key, now_token.room_key) rooms_changed.update(event.room_id for event in member_events) stream_ordering = RoomStreamToken.parse_stream_token( from_token.room_key).stream possibly_changed = set(changed) possibly_left = set() for room_id in rooms_changed: current_state_ids = yield self.store.get_current_state_ids(room_id) # The user may have left the room # TODO: Check if they actually did or if we were just invited. if room_id not in room_ids: for key, event_id in iteritems(current_state_ids): etype, state_key = key if etype != EventTypes.Member: continue possibly_left.add(state_key) continue # Fetch the current state at the time. try: event_ids = yield self.store.get_forward_extremeties_for_room( room_id, stream_ordering=stream_ordering) except errors.StoreError: # we have purged the stream_ordering index since the stream # ordering: treat it the same as a new room event_ids = [] # special-case for an empty prev state: include all members # in the changed list if not event_ids: for key, event_id in iteritems(current_state_ids): etype, state_key = key if etype != EventTypes.Member: continue possibly_changed.add(state_key) continue current_member_id = current_state_ids.get( (EventTypes.Member, user_id)) if not current_member_id: continue # mapping from event_id -> state_dict prev_state_ids = yield self.store.get_state_ids_for_events( event_ids) # Check if we've joined the room? If so we just blindly add all the users to # the "possibly changed" users. for state_dict in itervalues(prev_state_ids): member_event = state_dict.get((EventTypes.Member, user_id), None) if not member_event or member_event != current_member_id: for key, event_id in iteritems(current_state_ids): etype, state_key = key if etype != EventTypes.Member: continue possibly_changed.add(state_key) break # If there has been any change in membership, include them in the # possibly changed list. We'll check if they are joined below, # and we're not toooo worried about spuriously adding users. for key, event_id in iteritems(current_state_ids): etype, state_key = key if etype != EventTypes.Member: continue # check if this member has changed since any of the extremities # at the stream_ordering, and add them to the list if so. for state_dict in itervalues(prev_state_ids): prev_event_id = state_dict.get(key, None) if not prev_event_id or prev_event_id != event_id: if state_key != user_id: possibly_changed.add(state_key) break if possibly_changed or possibly_left: users_who_share_room = yield self.store.get_users_who_share_room_with_user( user_id) # Take the intersection of the users whose devices may have changed # and those that actually still share a room with the user possibly_joined = possibly_changed & users_who_share_room possibly_left = (possibly_changed | possibly_left) - users_who_share_room else: possibly_joined = [] possibly_left = [] defer.returnValue({ "changed": list(possibly_joined), "left": list(possibly_left), })
def get_user_ids_changed(self, user_id, from_token): """Get list of users that have had the devices updated, or have newly joined a room, that `user_id` may be interested in. Args: user_id (str) from_token (StreamToken) """ now_token = yield self.hs.get_event_sources().get_current_token() room_ids = yield self.store.get_rooms_for_user(user_id) # First we check if any devices have changed changed = yield self.store.get_user_whose_devices_changed( from_token.device_list_key ) # Then work out if any users have since joined rooms_changed = self.store.get_rooms_that_changed(room_ids, from_token.room_key) member_events = yield self.store.get_membership_changes_for_user( user_id, from_token.room_key, now_token.room_key ) rooms_changed.update(event.room_id for event in member_events) stream_ordering = RoomStreamToken.parse_stream_token( from_token.room_key ).stream possibly_changed = set(changed) possibly_left = set() for room_id in rooms_changed: current_state_ids = yield self.store.get_current_state_ids(room_id) # The user may have left the room # TODO: Check if they actually did or if we were just invited. if room_id not in room_ids: for key, event_id in iteritems(current_state_ids): etype, state_key = key if etype != EventTypes.Member: continue possibly_left.add(state_key) continue # Fetch the current state at the time. try: event_ids = yield self.store.get_forward_extremeties_for_room( room_id, stream_ordering=stream_ordering ) except errors.StoreError: # we have purged the stream_ordering index since the stream # ordering: treat it the same as a new room event_ids = [] # special-case for an empty prev state: include all members # in the changed list if not event_ids: for key, event_id in iteritems(current_state_ids): etype, state_key = key if etype != EventTypes.Member: continue possibly_changed.add(state_key) continue current_member_id = current_state_ids.get((EventTypes.Member, user_id)) if not current_member_id: continue # mapping from event_id -> state_dict prev_state_ids = yield self.store.get_state_ids_for_events(event_ids) # Check if we've joined the room? If so we just blindly add all the users to # the "possibly changed" users. for state_dict in itervalues(prev_state_ids): member_event = state_dict.get((EventTypes.Member, user_id), None) if not member_event or member_event != current_member_id: for key, event_id in iteritems(current_state_ids): etype, state_key = key if etype != EventTypes.Member: continue possibly_changed.add(state_key) break # If there has been any change in membership, include them in the # possibly changed list. We'll check if they are joined below, # and we're not toooo worried about spuriously adding users. for key, event_id in iteritems(current_state_ids): etype, state_key = key if etype != EventTypes.Member: continue # check if this member has changed since any of the extremities # at the stream_ordering, and add them to the list if so. for state_dict in itervalues(prev_state_ids): prev_event_id = state_dict.get(key, None) if not prev_event_id or prev_event_id != event_id: if state_key != user_id: possibly_changed.add(state_key) break if possibly_changed or possibly_left: users_who_share_room = yield self.store.get_users_who_share_room_with_user( user_id ) # Take the intersection of the users whose devices may have changed # and those that actually still share a room with the user possibly_joined = possibly_changed & users_who_share_room possibly_left = (possibly_changed | possibly_left) - users_who_share_room else: possibly_joined = [] possibly_left = [] defer.returnValue({ "changed": list(possibly_joined), "left": list(possibly_left), })
def get_room_events_stream_for_room( self, room_id, from_key, to_key, limit=0, order='DESC' ): """Get new room events in stream ordering since `from_key`. Args: room_id (str) from_key (str): Token from which no events are returned before to_key (str): Token from which no events are returned after. (This is typically the current stream token) limit (int): Maximum number of events to return order (str): Either "DESC" or "ASC". Determines which events are returned when the result is limited. If "DESC" then the most recent `limit` events are returned, otherwise returns the oldest `limit` events. Returns: Deferred[tuple[list[FrozenEvent], str]]: Returns the list of events (in ascending order) and the token from the start of the chunk of events returned. """ if from_key == to_key: defer.returnValue(([], from_key)) from_id = RoomStreamToken.parse_stream_token(from_key).stream to_id = RoomStreamToken.parse_stream_token(to_key).stream has_changed = yield self._events_stream_cache.has_entity_changed( room_id, from_id ) if not has_changed: defer.returnValue(([], from_key)) def f(txn): sql = ( "SELECT event_id, stream_ordering FROM events WHERE" " room_id = ?" " AND not outlier" " AND stream_ordering > ? AND stream_ordering <= ?" " ORDER BY stream_ordering %s LIMIT ?" ) % (order,) txn.execute(sql, (room_id, from_id, to_id, limit)) rows = [_EventDictReturn(row[0], None, row[1]) for row in txn] return rows rows = yield self.runInteraction("get_room_events_stream_for_room", f) ret = yield self.get_events_as_list([ r.event_id for r in rows], get_prev_content=True, ) self._set_before_and_after(ret, rows, topo_order=from_id is None) if order.lower() == "desc": ret.reverse() if rows: key = "s%d" % min(r.stream_ordering for r in rows) else: # Assume we didn't get anything because there was nothing to # get. key = from_key defer.returnValue((ret, key))
def get_user_ids_changed(self, user_id, from_token): """Get list of users that have had the devices updated, or have newly joined a room, that `user_id` may be interested in. Args: user_id (str) from_token (StreamToken) """ room_ids = yield self.store.get_rooms_for_user(user_id) # First we check if any devices have changed changed = yield self.store.get_user_whose_devices_changed( from_token.device_list_key ) # Then work out if any users have since joined rooms_changed = self.store.get_rooms_that_changed(room_ids, from_token.room_key) stream_ordering = RoomStreamToken.parse_stream_token( from_token.room_key).stream possibly_changed = set(changed) for room_id in rooms_changed: # Fetch the current state at the time. try: event_ids = yield self.store.get_forward_extremeties_for_room( room_id, stream_ordering=stream_ordering ) except errors.StoreError: # we have purged the stream_ordering index since the stream # ordering: treat it the same as a new room event_ids = [] current_state_ids = yield self.store.get_current_state_ids(room_id) # special-case for an empty prev state: include all members # in the changed list if not event_ids: for key, event_id in current_state_ids.iteritems(): etype, state_key = key if etype != EventTypes.Member: continue possibly_changed.add(state_key) continue # mapping from event_id -> state_dict prev_state_ids = yield self.store.get_state_ids_for_events(event_ids) # If there has been any change in membership, include them in the # possibly changed list. We'll check if they are joined below, # and we're not toooo worried about spuriously adding users. for key, event_id in current_state_ids.iteritems(): etype, state_key = key if etype != EventTypes.Member: continue # check if this member has changed since any of the extremities # at the stream_ordering, and add them to the list if so. for state_dict in prev_state_ids.values(): prev_event_id = state_dict.get(key, None) if not prev_event_id or prev_event_id != event_id: possibly_changed.add(state_key) break users_who_share_room = yield self.store.get_users_who_share_room_with_user( user_id ) # Take the intersection of the users whose devices may have changed # and those that actually still share a room with the user defer.returnValue(users_who_share_room & possibly_changed)
def get_room_events_stream_for_room(self, room_id, from_key, to_key, limit=0, order='DESC'): """Get new room events in stream ordering since `from_key`. Args: room_id (str) from_key (str): Token from which no events are returned before to_key (str): Token from which no events are returned after. (This is typically the current stream token) limit (int): Maximum number of events to return order (str): Either "DESC" or "ASC". Determines which events are returned when the result is limited. If "DESC" then the most recent `limit` events are returned, otherwise returns the oldest `limit` events. Returns: Deferred[tuple[list[FrozenEvent], str]]: Returns the list of events (in ascending order) and the token from the start of the chunk of events returned. """ if from_key == to_key: defer.returnValue(([], from_key)) from_id = RoomStreamToken.parse_stream_token(from_key).stream to_id = RoomStreamToken.parse_stream_token(to_key).stream has_changed = yield self._events_stream_cache.has_entity_changed( room_id, from_id ) if not has_changed: defer.returnValue(([], from_key)) def f(txn): sql = ( "SELECT event_id, stream_ordering FROM events WHERE" " room_id = ?" " AND not outlier" " AND stream_ordering > ? AND stream_ordering <= ?" " ORDER BY stream_ordering %s LIMIT ?" ) % (order,) txn.execute(sql, (room_id, from_id, to_id, limit)) rows = [_EventDictReturn(row[0], None, row[1]) for row in txn] return rows rows = yield self.runInteraction("get_room_events_stream_for_room", f) ret = yield self._get_events( [r.event_id for r in rows], get_prev_content=True ) self._set_before_and_after(ret, rows, topo_order=from_id is None) if order.lower() == "desc": ret.reverse() if rows: key = "s%d" % min(r.stream_ordering for r in rows) else: # Assume we didn't get anything because there was nothing to # get. key = from_key defer.returnValue((ret, key))
def get_room_events_stream(self, user_id, from_key, to_key, room_id, limit=0, with_feedback=False): # TODO (erikj): Handle compressed feedback current_room_membership_sql = ( "SELECT m.room_id FROM room_memberships as m " " INNER JOIN current_state_events as c" " ON m.event_id = c.event_id AND c.state_key = m.user_id" " WHERE m.user_id = ? AND m.membership = 'join'" ) # We also want to get any membership events about that user, e.g. # invites or leave notifications. membership_sql = ( "SELECT m.event_id FROM room_memberships as m " "INNER JOIN current_state_events as c ON m.event_id = c.event_id " "WHERE m.user_id = ? " ) if limit: limit = max(limit, MAX_STREAM_SIZE) else: limit = MAX_STREAM_SIZE # From and to keys should be integers from ordering. from_id = RoomStreamToken.parse_stream_token(from_key) to_id = RoomStreamToken.parse_stream_token(to_key) if from_key == to_key: return defer.succeed(([], to_key)) sql = ( "SELECT e.event_id, e.stream_ordering FROM events AS e WHERE " "(e.outlier = ? AND (room_id IN (%(current)s)) OR " "(event_id IN (%(invites)s))) " "AND e.stream_ordering > ? AND e.stream_ordering <= ? " "ORDER BY stream_ordering ASC LIMIT %(limit)d " ) % { "current": current_room_membership_sql, "invites": membership_sql, "limit": limit } def f(txn): txn.execute(sql, (False, user_id, user_id, from_id.stream, to_id.stream,)) rows = self.cursor_to_dict(txn) ret = self._get_events_txn( txn, [r["event_id"] for r in rows], get_prev_content=True ) self._set_before_and_after(ret, rows) if rows: key = "s%d" % max(r["stream_ordering"] for r in rows) else: # Assume we didn't get anything because there was nothing to # get. key = to_key return ret, key return self.runInteraction("get_room_events_stream", f)
def get_room_events_stream_for_room(self, room_id, from_key, to_key, limit=0, order='DESC'): # Note: If from_key is None then we return in topological order. This # is because in that case we're using this as a "get the last few messages # in a room" function, rather than "get new messages since last sync" if from_key is not None: from_id = RoomStreamToken.parse_stream_token(from_key).stream else: from_id = None to_id = RoomStreamToken.parse_stream_token(to_key).stream if from_key == to_key: defer.returnValue(([], from_key)) if from_id: has_changed = yield self._events_stream_cache.has_entity_changed( room_id, from_id ) if not has_changed: defer.returnValue(([], from_key)) def f(txn): if from_id is not None: sql = ( "SELECT event_id, stream_ordering FROM events WHERE" " room_id = ?" " AND not outlier" " AND stream_ordering > ? AND stream_ordering <= ?" " ORDER BY stream_ordering %s LIMIT ?" ) % (order,) txn.execute(sql, (room_id, from_id, to_id, limit)) else: sql = ( "SELECT event_id, stream_ordering FROM events WHERE" " room_id = ?" " AND not outlier" " AND stream_ordering <= ?" " ORDER BY topological_ordering %s, stream_ordering %s LIMIT ?" ) % (order, order,) txn.execute(sql, (room_id, to_id, limit)) rows = self.cursor_to_dict(txn) return rows rows = yield self.runInteraction("get_room_events_stream_for_room", f) ret = yield self._get_events( [r["event_id"] for r in rows], get_prev_content=True ) self._set_before_and_after(ret, rows, topo_order=from_id is None) if order.lower() == "desc": ret.reverse() if rows: key = "s%d" % min(r["stream_ordering"] for r in rows) else: # Assume we didn't get anything because there was nothing to # get. key = from_key defer.returnValue((ret, key))
def get_appservice_room_stream(self, service, from_key, to_key, limit=0): # NB this lives here instead of appservice.py so we can reuse the # 'private' StreamToken class in this file. if limit: limit = max(limit, MAX_STREAM_SIZE) else: limit = MAX_STREAM_SIZE # From and to keys should be integers from ordering. from_id = RoomStreamToken.parse_stream_token(from_key) to_id = RoomStreamToken.parse_stream_token(to_key) if from_key == to_key: defer.returnValue(([], to_key)) return # select all the events between from/to with a sensible limit sql = ( "SELECT e.event_id, e.room_id, e.type, s.state_key, " "e.stream_ordering FROM events AS e " "LEFT JOIN state_events as s ON " "e.event_id = s.event_id " "WHERE e.stream_ordering > ? AND e.stream_ordering <= ? " "ORDER BY stream_ordering ASC LIMIT %(limit)d " ) % { "limit": limit } def f(txn): # pull out all the events between the tokens txn.execute(sql, (from_id.stream, to_id.stream,)) rows = self.cursor_to_dict(txn) # Logic: # - We want ALL events which match the AS room_id regex # - We want ALL events which match the rooms represented by the AS # room_alias regex # - We want ALL events for rooms that AS users have joined. # This is currently supported via get_app_service_rooms (which is # used for the Notifier listener rooms). We can't reasonably make a # SQL query for these room IDs, so we'll pull all the events between # from/to and filter in python. rooms_for_as = self._get_app_service_rooms_txn(txn, service) room_ids_for_as = [r.room_id for r in rooms_for_as] def app_service_interested(row): if row["room_id"] in room_ids_for_as: return True if row["type"] == EventTypes.Member: if service.is_interested_in_user(row.get("state_key")): return True return False return [r for r in rows if app_service_interested(r)] rows = yield self.runInteraction("get_appservice_room_stream", f) ret = yield self._get_events( [r["event_id"] for r in rows], get_prev_content=True ) self._set_before_and_after(ret, rows, topo_order=from_id is None) if rows: key = "s%d" % max(r["stream_ordering"] for r in rows) else: # Assume we didn't get anything because there was nothing to # get. key = to_key defer.returnValue((ret, key))
def get_room_events_stream_for_room(self, room_id, from_key, to_key, limit=0, order='DESC'): # Note: If from_key is None then we return in topological order. This # is because in that case we're using this as a "get the last few messages # in a room" function, rather than "get new messages since last sync" if from_key is not None: from_id = RoomStreamToken.parse_stream_token(from_key).stream else: from_id = None to_id = RoomStreamToken.parse_stream_token(to_key).stream if from_key == to_key: defer.returnValue(([], from_key)) if from_id: has_changed = yield self._events_stream_cache.has_entity_changed( room_id, from_id) if not has_changed: defer.returnValue(([], from_key)) def f(txn): if from_id is not None: sql = ("SELECT event_id, stream_ordering FROM events WHERE" " room_id = ?" " AND not outlier" " AND stream_ordering > ? AND stream_ordering <= ?" " ORDER BY stream_ordering %s LIMIT ?") % (order, ) txn.execute(sql, (room_id, from_id, to_id, limit)) else: sql = ( "SELECT event_id, stream_ordering FROM events WHERE" " room_id = ?" " AND not outlier" " AND stream_ordering <= ?" " ORDER BY topological_ordering %s, stream_ordering %s LIMIT ?" ) % ( order, order, ) txn.execute(sql, (room_id, to_id, limit)) rows = self.cursor_to_dict(txn) return rows rows = yield self.runInteraction("get_room_events_stream_for_room", f) ret = yield self._get_events([r["event_id"] for r in rows], get_prev_content=True) self._set_before_and_after(ret, rows, topo_order=from_id is None) if order.lower() == "desc": ret.reverse() if rows: key = "s%d" % min(r["stream_ordering"] for r in rows) else: # Assume we didn't get anything because there was nothing to # get. key = from_key defer.returnValue((ret, key))
def get_recent_events_for_room(self, room_id, limit, end_token, from_token=None): # TODO (erikj): Handle compressed feedback end_token = RoomStreamToken.parse_stream_token(end_token) if from_token is None: sql = ( "SELECT stream_ordering, topological_ordering, event_id" " FROM events" " WHERE room_id = ? AND stream_ordering <= ? AND outlier = ?" " ORDER BY topological_ordering DESC, stream_ordering DESC" " LIMIT ?" ) else: from_token = RoomStreamToken.parse_stream_token(from_token) sql = ( "SELECT stream_ordering, topological_ordering, event_id" " FROM events" " WHERE room_id = ? AND stream_ordering > ?" " AND stream_ordering <= ? AND outlier = ?" " ORDER BY topological_ordering DESC, stream_ordering DESC" " LIMIT ?" ) def get_recent_events_for_room_txn(txn): if from_token is None: txn.execute(sql, (room_id, end_token.stream, False, limit,)) else: txn.execute(sql, ( room_id, from_token.stream, end_token.stream, False, limit )) rows = self.cursor_to_dict(txn) rows.reverse() # As we selected with reverse ordering if rows: # Tokens are positions between events. # This token points *after* the last event in the chunk. # We need it to point to the event before it in the chunk # since we are going backwards so we subtract one from the # stream part. topo = rows[0]["topological_ordering"] toke = rows[0]["stream_ordering"] - 1 start_token = str(RoomStreamToken(topo, toke)) token = (start_token, str(end_token)) else: token = (str(end_token), str(end_token)) return rows, token rows, token = yield self.runInteraction( "get_recent_events_for_room", get_recent_events_for_room_txn ) logger.debug("stream before") events = yield self._get_events( [r["event_id"] for r in rows], get_prev_content=True ) logger.debug("stream after") self._set_before_and_after(events, rows) defer.returnValue((events, token))
def get_appservice_room_stream(self, service, from_key, to_key, limit=0): # NB this lives here instead of appservice.py so we can reuse the # 'private' StreamToken class in this file. if limit: limit = max(limit, MAX_STREAM_SIZE) else: limit = MAX_STREAM_SIZE # From and to keys should be integers from ordering. from_id = RoomStreamToken.parse_stream_token(from_key) to_id = RoomStreamToken.parse_stream_token(to_key) if from_key == to_key: defer.returnValue(([], to_key)) return # select all the events between from/to with a sensible limit sql = ("SELECT e.event_id, e.room_id, e.type, s.state_key, " "e.stream_ordering FROM events AS e " "LEFT JOIN state_events as s ON " "e.event_id = s.event_id " "WHERE e.stream_ordering > ? AND e.stream_ordering <= ? " "ORDER BY stream_ordering ASC LIMIT %(limit)d ") % { "limit": limit } def f(txn): # pull out all the events between the tokens txn.execute(sql, ( from_id.stream, to_id.stream, )) rows = self.cursor_to_dict(txn) # Logic: # - We want ALL events which match the AS room_id regex # - We want ALL events which match the rooms represented by the AS # room_alias regex # - We want ALL events for rooms that AS users have joined. # This is currently supported via get_app_service_rooms (which is # used for the Notifier listener rooms). We can't reasonably make a # SQL query for these room IDs, so we'll pull all the events between # from/to and filter in python. rooms_for_as = self._get_app_service_rooms_txn(txn, service) room_ids_for_as = [r.room_id for r in rooms_for_as] def app_service_interested(row): if row["room_id"] in room_ids_for_as: return True if row["type"] == EventTypes.Member: if service.is_interested_in_user(row.get("state_key")): return True return False return [r for r in rows if app_service_interested(r)] rows = yield self.runInteraction("get_appservice_room_stream", f) ret = yield self._get_events([r["event_id"] for r in rows], get_prev_content=True) self._set_before_and_after(ret, rows, topo_order=from_id is None) if rows: key = "s%d" % max(r["stream_ordering"] for r in rows) else: # Assume we didn't get anything because there was nothing to # get. key = to_key defer.returnValue((ret, key))
def get_room_events_stream( self, user_id, from_key, to_key, limit=0, is_guest=False, room_ids=None ): room_ids = room_ids or [] room_ids = [r for r in room_ids] if is_guest: current_room_membership_sql = ( "SELECT c.room_id FROM history_visibility AS h" " INNER JOIN current_state_events AS c" " ON h.event_id = c.event_id" " WHERE c.room_id IN (%s)" " AND h.history_visibility = 'world_readable'" % ( ",".join(map(lambda _: "?", room_ids)) ) ) current_room_membership_args = room_ids else: current_room_membership_sql = ( "SELECT m.room_id FROM room_memberships as m " " INNER JOIN current_state_events as c" " ON m.event_id = c.event_id AND c.state_key = m.user_id" " WHERE m.user_id = ? AND m.membership = 'join'" ) current_room_membership_args = [user_id] # We also want to get any membership events about that user, e.g. # invites or leave notifications. membership_sql = ( "SELECT m.event_id FROM room_memberships as m " "INNER JOIN current_state_events as c ON m.event_id = c.event_id " "WHERE m.user_id = ? " ) membership_args = [user_id] if limit: limit = max(limit, MAX_STREAM_SIZE) else: limit = MAX_STREAM_SIZE # From and to keys should be integers from ordering. from_id = RoomStreamToken.parse_stream_token(from_key) to_id = RoomStreamToken.parse_stream_token(to_key) if from_key == to_key: return defer.succeed(([], to_key)) sql = ( "SELECT e.event_id, e.stream_ordering FROM events AS e WHERE " "(e.outlier = ? AND (room_id IN (%(current)s)) OR " "(event_id IN (%(invites)s))) " "AND e.stream_ordering > ? AND e.stream_ordering <= ? " "ORDER BY stream_ordering ASC LIMIT %(limit)d " ) % { "current": current_room_membership_sql, "invites": membership_sql, "limit": limit } def f(txn): args = ([False] + current_room_membership_args + membership_args + [from_id.stream, to_id.stream]) txn.execute(sql, args) rows = self.cursor_to_dict(txn) ret = self._get_events_txn( txn, [r["event_id"] for r in rows], get_prev_content=True ) self._set_before_and_after(ret, rows) if rows: key = "s%d" % max(r["stream_ordering"] for r in rows) else: # Assume we didn't get anything because there was nothing to # get. key = to_key return ret, key return self.runInteraction("get_room_events_stream", f)