def _get_filtered_current_state_ids_txn(txn): results = {} sql = """SELECT type, state_key, event_id FROM current_state_events WHERE room_id = ? %s""" # Turns out that postgres doesn't like doing a list of OR's and # is about 1000x slower, so we just issue a query for each specific # type seperately. if types: clause_to_args = [("AND type = ? AND state_key = ?", (etype, state_key)) if state_key is not None else ("AND type = ?", (etype, )) for etype, state_key in types] if include_other_types: unique_types = set(filtered_types) clause_to_args.append( ("AND type <> ? " * len(unique_types), list(unique_types))) else: # If types is None we fetch all the state, and so just use an # empty where clause with no extra args. clause_to_args = [("", [])] for where_clause, where_args in clause_to_args: args = [room_id] args.extend(where_args) txn.execute(sql % (where_clause, ), args) for row in txn: typ, state_key, event_id = row key = (intern_string(typ), intern_string(state_key)) results[key] = event_id return results
def _get_current_state_ids_txn(txn): txn.execute( """SELECT type, state_key, event_id FROM current_state_events WHERE room_id = ? """, (room_id, )) return {(intern_string(r[0]), intern_string(r[1])): to_ascii(r[2]) for r in txn}
def _get_state_for_groups(self, groups, types=None): """Given list of groups returns dict of group -> list of state events with matching types. `types` is a list of `(type, state_key)`, where a `state_key` of None matches all state_keys. If `types` is None then all events are returned. """ if types: types = frozenset(types) results = {} missing_groups = [] if types is not None: for group in set(groups): state_dict_ids, _, got_all = self._get_some_state_from_cache( group, types ) results[group] = state_dict_ids if not got_all: missing_groups.append(group) else: for group in set(groups): state_dict_ids, got_all = self._get_all_state_from_cache( group ) results[group] = state_dict_ids if not got_all: missing_groups.append(group) if missing_groups: # Okay, so we have some missing_types, lets fetch them. cache_seq_num = self._state_group_cache.sequence group_to_state_dict = yield self._get_state_groups_from_groups( missing_groups, types ) # Now we want to update the cache with all the things we fetched # from the database. for group, group_state_dict in group_to_state_dict.iteritems(): state_dict = results[group] state_dict.update( ((intern_string(k[0]), intern_string(k[1])), to_ascii(v)) for k, v in group_state_dict.iteritems() ) self._state_group_cache.update( cache_seq_num, key=group, value=state_dict, full=(types is None), known_absent=types, ) defer.returnValue(results)
def _get_current_state_ids_txn(txn): txn.execute( """SELECT type, state_key, event_id FROM current_state_events WHERE room_id = ? """, (room_id,) ) return { (intern_string(r[0]), intern_string(r[1])): to_ascii(r[2]) for r in txn }
def _get_current_state_ids_txn( txn: LoggingTransaction) -> StateMap[str]: txn.execute( """SELECT type, state_key, event_id FROM current_state_events WHERE room_id = ? """, (room_id, ), ) return {(intern_string(r[0]), intern_string(r[1])): r[2] for r in txn}
async def get_destinations(self, state_entry: "_StateCacheEntry") -> Set[str]: """Get set of destinations for a state entry Args: state_entry Returns: The destinations as a set. """ if state_entry.state_group == self.state_group: return frozenset(self.hosts_to_joined_users) with (await self.linearizer.queue(())): if state_entry.state_group == self.state_group: pass elif state_entry.prev_group == self.state_group: for (typ, state_key), event_id in state_entry.delta_ids.items(): if typ != EventTypes.Member: continue host = intern_string(get_domain_from_id(state_key)) user_id = state_key known_joins = self.hosts_to_joined_users.setdefault( host, set()) event = await self.store.get_event(event_id) if event.membership == Membership.JOIN: known_joins.add(user_id) else: known_joins.discard(user_id) if not known_joins: self.hosts_to_joined_users.pop(host, None) else: joined_users = await self.store.get_joined_users_from_state( self.room_id, state_entry) self.hosts_to_joined_users = {} for user_id in joined_users: host = intern_string(get_domain_from_id(user_id)) self.hosts_to_joined_users.setdefault(host, set()).add(user_id) if state_entry.state_group: self.state_group = state_entry.state_group else: self.state_group = object() self._len = sum( len(v) for v in self.hosts_to_joined_users.values()) return frozenset(self.hosts_to_joined_users)
def _get_state_for_groups(self, groups, types=None): """Given list of groups returns dict of group -> list of state events with matching types. `types` is a list of `(type, state_key)`, where a `state_key` of None matches all state_keys. If `types` is None then all events are returned. """ if types: types = frozenset(types) results = {} missing_groups = [] if types is not None: for group in set(groups): state_dict_ids, _, got_all = self._get_some_state_from_cache( group, types) results[group] = state_dict_ids if not got_all: missing_groups.append(group) else: for group in set(groups): state_dict_ids, got_all = self._get_all_state_from_cache(group) results[group] = state_dict_ids if not got_all: missing_groups.append(group) if missing_groups: # Okay, so we have some missing_types, lets fetch them. cache_seq_num = self._state_group_cache.sequence group_to_state_dict = yield self._get_state_groups_from_groups( missing_groups, types) # Now we want to update the cache with all the things we fetched # from the database. for group, group_state_dict in iteritems(group_to_state_dict): state_dict = results[group] state_dict.update( ((intern_string(k[0]), intern_string(k[1])), to_ascii(v)) for k, v in iteritems(group_state_dict)) self._state_group_cache.update( cache_seq_num, key=group, value=state_dict, full=(types is None), known_absent=types, ) defer.returnValue(results)
def get_destinations(self, state_entry): """Get set of destinations for a state entry Args: state_entry(synapse.state._StateCacheEntry) """ if state_entry.state_group == self.state_group: defer.returnValue(frozenset(self.hosts_to_joined_users)) with (yield self.linearizer.queue(())): if state_entry.state_group == self.state_group: pass elif state_entry.prev_group == self.state_group: for (typ, state_key), event_id in iteritems(state_entry.delta_ids): if typ != EventTypes.Member: continue host = intern_string(get_domain_from_id(state_key)) user_id = state_key known_joins = self.hosts_to_joined_users.setdefault( host, set()) event = yield self.store.get_event(event_id) if event.membership == Membership.JOIN: known_joins.add(user_id) else: known_joins.discard(user_id) if not known_joins: self.hosts_to_joined_users.pop(host, None) else: joined_users = yield self.store.get_joined_users_from_state( self.room_id, state_entry, ) self.hosts_to_joined_users = {} for user_id in joined_users: host = intern_string(get_domain_from_id(user_id)) self.hosts_to_joined_users.setdefault(host, set()).add(user_id) if state_entry.state_group: self.state_group = state_entry.state_group else: self.state_group = object() self._len = sum( len(v) for v in itervalues(self.hosts_to_joined_users)) defer.returnValue(frozenset(self.hosts_to_joined_users))
def _get_joined_hosts(self, room_id, state_group, current_state_ids): # We don't use `state_group`, its there so that we can cache based # on it. However, its important that its never None, since two current_state's # with a state_group of None are likely to be different. # See bulk_get_push_rules_for_room for how we work around this. assert state_group is not None joined_hosts = set() for etype, state_key in current_state_ids: if etype == EventTypes.Member: try: host = get_domain_from_id(state_key) except: logger.warn("state_key not user_id: %s", state_key) continue if host in joined_hosts: continue event_id = current_state_ids[(etype, state_key)] event = yield self.get_event(event_id, allow_none=True) if event and event.content["membership"] == Membership.JOIN: joined_hosts.add(intern_string(host)) defer.returnValue(joined_hosts)
def get_destinations(self, state_entry): """Get set of destinations for a state entry Args: state_entry(synapse.state._StateCacheEntry) """ if state_entry.state_group == self.state_group: defer.returnValue(frozenset(self.hosts_to_joined_users)) with (yield self.linearizer.queue(())): if state_entry.state_group == self.state_group: pass elif state_entry.prev_group == self.state_group: for (typ, state_key), event_id in iteritems(state_entry.delta_ids): if typ != EventTypes.Member: continue host = intern_string(get_domain_from_id(state_key)) user_id = state_key known_joins = self.hosts_to_joined_users.setdefault(host, set()) event = yield self.store.get_event(event_id) if event.membership == Membership.JOIN: known_joins.add(user_id) else: known_joins.discard(user_id) if not known_joins: self.hosts_to_joined_users.pop(host, None) else: joined_users = yield self.store.get_joined_users_from_state( self.room_id, state_entry ) self.hosts_to_joined_users = {} for user_id in joined_users: host = intern_string(get_domain_from_id(user_id)) self.hosts_to_joined_users.setdefault(host, set()).add(user_id) if state_entry.state_group: self.state_group = state_entry.state_group else: self.state_group = object() self._len = sum(len(v) for v in itervalues(self.hosts_to_joined_users)) defer.returnValue(frozenset(self.hosts_to_joined_users))
def _get_filtered_current_state_ids_txn(txn): results = {} sql = """ SELECT type, state_key, event_id FROM current_state_events WHERE room_id = ? """ if where_clause: sql += " AND (%s)" % (where_clause, ) args = [room_id] args.extend(where_args) txn.execute(sql, args) for row in txn: typ, state_key, event_id = row key = (intern_string(typ), intern_string(state_key)) results[key] = event_id return results
def _get_state_for_groups(self, groups, types=None): """Given list of groups returns dict of group -> list of state events with matching types. `types` is a list of `(type, state_key)`, where a `state_key` of None matches all state_keys. If `types` is None then all events are returned. """ if types: types = frozenset(types) results = {} missing_groups = [] if types is not None: for group in set(groups): state_dict_ids, missing_types, got_all = self._get_some_state_from_cache( group, types ) results[group] = state_dict_ids if not got_all: missing_groups.append(group) else: for group in set(groups): state_dict_ids, got_all = self._get_all_state_from_cache( group ) results[group] = state_dict_ids if not got_all: missing_groups.append(group) if missing_groups: # Okay, so we have some missing_types, lets fetch them. cache_seq_num = self._state_group_cache.sequence group_to_state_dict = yield self._get_state_groups_from_groups( missing_groups, types ) # Now we want to update the cache with all the things we fetched # from the database. for group, group_state_dict in group_to_state_dict.items(): if types: # We delibrately put key -> None mappings into the cache to # cache absence of the key, on the assumption that if we've # explicitly asked for some types then we will probably ask # for them again. state_dict = { (intern_string(etype), intern_string(state_key)): None for (etype, state_key) in types } state_dict.update(results[group]) results[group] = state_dict else: state_dict = results[group] state_dict.update(group_state_dict) self._state_group_cache.update( cache_seq_num, key=group, value=state_dict, full=(types is None), ) state_events = yield self._get_events( [ev_id for sd in results.values() for ev_id in sd.values()], get_prev_content=False ) state_events = {e.event_id: e for e in state_events} # Remove all the entries with None values. The None values were just # used for bookkeeping in the cache. for group, state_dict in results.items(): results[group] = { key: state_events[event_id] for key, event_id in state_dict.items() if event_id and event_id in state_events } defer.returnValue(results)
async def _get_joined_hosts( self, room_id: str, state_group: int, current_state_ids: StateMap[str], state_entry: "_StateCacheEntry", ) -> FrozenSet[str]: # We don't use `state_group`, its there so that we can cache based on # it. However, its important that its never None, since two # current_state's with a state_group of None are likely to be different. # # The `state_group` must match the `state_entry.state_group` (if not None). assert state_group is not None assert state_entry.state_group is None or state_entry.state_group == state_group # We use a secondary cache of previous work to allow us to build up the # joined hosts for the given state group based on previous state groups. # # We cache one object per room containing the results of the last state # group we got joined hosts for. The idea is that generally # `get_joined_hosts` is called with the "current" state group for the # room, and so consecutive calls will be for consecutive state groups # which point to the previous state group. cache = await self._get_joined_hosts_cache(room_id) # If the state group in the cache matches, we already have the data we need. if state_entry.state_group == cache.state_group: return frozenset(cache.hosts_to_joined_users) # Since we'll mutate the cache we need to lock. with (await self._joined_host_linearizer.queue(room_id)): if state_entry.state_group == cache.state_group: # Same state group, so nothing to do. We've already checked for # this above, but the cache may have changed while waiting on # the lock. pass elif state_entry.prev_group == cache.state_group: # The cached work is for the previous state group, so we work out # the delta. for (typ, state_key), event_id in state_entry.delta_ids.items(): if typ != EventTypes.Member: continue host = intern_string(get_domain_from_id(state_key)) user_id = state_key known_joins = cache.hosts_to_joined_users.setdefault( host, set()) event = await self.get_event(event_id) if event.membership == Membership.JOIN: known_joins.add(user_id) else: known_joins.discard(user_id) if not known_joins: cache.hosts_to_joined_users.pop(host, None) else: # The cache doesn't match the state group or prev state group, # so we calculate the result from first principles. joined_users = await self.get_joined_users_from_state( room_id, state_entry) cache.hosts_to_joined_users = {} for user_id in joined_users: host = intern_string(get_domain_from_id(user_id)) cache.hosts_to_joined_users.setdefault(host, set()).add(user_id) if state_entry.state_group: cache.state_group = state_entry.state_group else: cache.state_group = object() return frozenset(cache.hosts_to_joined_users)