def _get_state_for_groups(self, groups: Iterable[int], state_filter: StateFilter = StateFilter.all()): """Gets the state at each of a list of state groups, optionally filtering by type/state_key Args: groups: list of state groups for which we want to get the state. state_filter: The state filter used to fetch state from the database. Returns: Deferred[Dict[int, StateMap[str]]]: Dict of state group to state map. """ member_filter, non_member_filter = state_filter.get_member_split() # Now we look them up in the member and non-member caches ( non_member_state, incomplete_groups_nm, ) = yield self._get_state_for_groups_using_cache( groups, self._state_group_cache, state_filter=non_member_filter) ( member_state, incomplete_groups_m, ) = yield self._get_state_for_groups_using_cache( groups, self._state_group_members_cache, state_filter=member_filter) state = dict(non_member_state) for group in groups: state[group].update(member_state[group]) # Now fetch any missing groups from the database incomplete_groups = incomplete_groups_m | incomplete_groups_nm if not incomplete_groups: return state cache_sequence_nm = self._state_group_cache.sequence cache_sequence_m = self._state_group_members_cache.sequence # Help the cache hit ratio by expanding the filter a bit db_state_filter = state_filter.return_expanded() group_to_state_dict = yield self._get_state_groups_from_groups( list(incomplete_groups), state_filter=db_state_filter) # Now lets update the caches self._insert_into_cache( group_to_state_dict, db_state_filter, cache_seq_num_members=cache_sequence_m, cache_seq_num_non_members=cache_sequence_nm, ) # And finally update the result dict, by filtering out any extra # stuff we pulled out of the database. for group, group_state_dict in iteritems(group_to_state_dict): # We just replace any existing entries, as we will have loaded # everything we need from the database anyway. state[group] = state_filter.filter_state(group_state_dict) return state
def _insert_into_cache( self, group_to_state_dict: Dict[int, StateMap[str]], state_filter: StateFilter, cache_seq_num_members: int, cache_seq_num_non_members: int, ) -> None: """Inserts results from querying the database into the relevant cache. Args: group_to_state_dict: The new entries pulled from database. Map from state group to state dict state_filter: The state filter used to fetch state from the database. cache_seq_num_members: Sequence number of member cache since last lookup in cache cache_seq_num_non_members: Sequence number of member cache since last lookup in cache """ # We need to work out which types we've fetched from the DB for the # member vs non-member caches. This should be as accurate as possible, # but can be an underestimate (e.g. when we have wild cards) member_filter, non_member_filter = state_filter.get_member_split() if member_filter.is_full(): # We fetched all member events member_types = None else: # `concrete_types()` will only return a subset when there are wild # cards in the filter, but that's fine. member_types = member_filter.concrete_types() if non_member_filter.is_full(): # We fetched all non member events non_member_types = None else: non_member_types = non_member_filter.concrete_types() for group, group_state_dict in group_to_state_dict.items(): state_dict_members = {} state_dict_non_members = {} for k, v in group_state_dict.items(): if k[0] == EventTypes.Member: state_dict_members[k] = v else: state_dict_non_members[k] = v self._state_group_members_cache.update( cache_seq_num_members, key=group, value=state_dict_members, fetched_keys=member_types, ) self._state_group_cache.update( cache_seq_num_non_members, key=group, value=state_dict_non_members, fetched_keys=non_member_types, )