def _invalidate_state_caches_and_stream(self, txn, room_id,
members_changed):
"""Special case invalidation of caches based on current state.
We special case this so that we can batch the cache invalidations into a
single replication poke.
Args:
txn
room_id (str): Room where state changed
members_changed (iterable[str]): The user_ids of members that have changed
"""
txn.call_after(self._invalidate_state_caches, room_id, members_changed)
if members_changed:
# We need to be careful that the size of the `members_changed` list
# isn't so large that it causes problems sending over replication, so we
# send them in chunks.
# Max line length is 16K, and max user ID length is 255, so 50 should
# be safe.
for chunk in batch_iter(members_changed, 50):
keys = itertools.chain([room_id], chunk)
self._send_invalidation_to_replication(
txn, CURRENT_STATE_CACHE_NAME, keys)
else:
# if no members changed, we still need to invalidate the other caches.
self._send_invalidation_to_replication(txn,
CURRENT_STATE_CACHE_NAME,
[room_id])
def _update_presence_txn(self, txn, stream_orderings, presence_states):
for stream_id, state in zip(stream_orderings, presence_states):
txn.call_after(
self.presence_stream_cache.entity_has_changed,
state.user_id,
stream_id,
)
txn.call_after(self._get_presence_for_user.invalidate,
(state.user_id, ))
# Actually insert new rows
self._simple_insert_many_txn(
txn,
table="presence_stream",
values=[{
"stream_id": stream_id,
"user_id": state.user_id,
"state": state.state,
"last_active_ts": state.last_active_ts,
"last_federation_update_ts": state.last_federation_update_ts,
"last_user_sync_ts": state.last_user_sync_ts,
"status_msg": state.status_msg,
"currently_active": state.currently_active,
} for state in presence_states],
)
# Delete old rows to stop database from getting really big
sql = ("DELETE FROM presence_stream WHERE"
" stream_id < ?"
" AND user_id IN (%s)")
for states in batch_iter(presence_states, 50):
args = [stream_id]
args.extend(s.user_id for s in states)
txn.execute(sql % (",".join("?" for _ in states), ), args)
def _update_presence_txn(self, txn, stream_orderings, presence_states):
for stream_id, state in zip(stream_orderings, presence_states):
txn.call_after(self.presence_stream_cache.entity_has_changed,
state.user_id, stream_id)
txn.call_after(self._get_presence_for_user.invalidate,
(state.user_id, ))
# Actually insert new rows
self.db.simple_insert_many_txn(
txn,
table="presence_stream",
values=[{
"stream_id": stream_id,
"user_id": state.user_id,
"state": state.state,
"last_active_ts": state.last_active_ts,
"last_federation_update_ts": state.last_federation_update_ts,
"last_user_sync_ts": state.last_user_sync_ts,
"status_msg": state.status_msg,
"currently_active": state.currently_active,
} for state in presence_states],
)
# Delete old rows to stop database from getting really big
sql = "DELETE FROM presence_stream WHERE stream_id < ? AND "
for states in batch_iter(presence_states, 50):
clause, args = make_in_list_sql_clause(self.database_engine,
"user_id",
[s.user_id for s in states])
txn.execute(sql + clause, [stream_id] + list(args))
def _get_users_whose_devices_changed_txn(txn):
changes = set()
sql = """
SELECT DISTINCT user_id FROM device_lists_stream
WHERE stream_id > ?
AND user_id IN (%s)
"""
for chunk in batch_iter(to_check, 100):
txn.execute(sql % (",".join("?" for _ in chunk),), (from_key,) + chunk)
changes.update(user_id for user_id, in txn)
return changes
def _get_users_whose_devices_changed_txn(txn):
changes = set()
sql = """
SELECT DISTINCT user_id FROM device_lists_stream
WHERE stream_id > ?
AND
"""
for chunk in batch_iter(to_check, 100):
clause, args = make_in_list_sql_clause(txn.database_engine,
"user_id", chunk)
txn.execute(sql + clause, (from_key, ) + tuple(args))
changes.update(user_id for user_id, in txn)
return changes
def _update_presence_txn(self, txn, stream_orderings, presence_states):
for stream_id, state in zip(stream_orderings, presence_states):
txn.call_after(
self.presence_stream_cache.entity_has_changed,
state.user_id, stream_id,
)
txn.call_after(
self._get_presence_for_user.invalidate, (state.user_id,)
)
# Actually insert new rows
self._simple_insert_many_txn(
txn,
table="presence_stream",
values=[
{
"stream_id": stream_id,
"user_id": state.user_id,
"state": state.state,
"last_active_ts": state.last_active_ts,
"last_federation_update_ts": state.last_federation_update_ts,
"last_user_sync_ts": state.last_user_sync_ts,
"status_msg": state.status_msg,
"currently_active": state.currently_active,
}
for state in presence_states
],
)
# Delete old rows to stop database from getting really big
sql = (
"DELETE FROM presence_stream WHERE"
" stream_id < ?"
" AND user_id IN (%s)"
)
for states in batch_iter(presence_states, 50):
args = [stream_id]
args.extend(s.user_id for s in states)
txn.execute(
sql % (",".join("?" for _ in states),),
args
)
def _fetch_event_rows(self, txn, event_ids):
"""Fetch event rows from the database
Events which are not found are omitted from the result.
The returned per-event dicts contain the following keys:
* event_id (str)
* json (str): json-encoded event structure
* internal_metadata (str): json-encoded internal metadata dict
* format_version (int|None): The format of the event. Hopefully one
of EventFormatVersions. 'None' means the event predates
EventFormatVersions (so the event is format V1).
* rejected_reason (str|None): if the event was rejected, the reason
why.
* redactions (List[str]): a list of event-ids which (claim to) redact
this event.
Args:
txn (twisted.enterprise.adbapi.Connection):
event_ids (Iterable[str]): event IDs to fetch
Returns:
Dict[str, Dict]: a map from event id to event info.
"""
event_dict = {}
for evs in batch_iter(event_ids, 200):
sql = ("SELECT "
" e.event_id, "
" e.internal_metadata,"
" e.json,"
" e.format_version, "
" rej.reason "
" FROM event_json as e"
" LEFT JOIN rejections as rej USING (event_id)"
" WHERE e.event_id IN (%s)") % (",".join(
["?"] * len(evs)), )
txn.execute(sql, evs)
for row in txn:
event_id = row[0]
event_dict[event_id] = {
"event_id": event_id,
"internal_metadata": row[1],
"json": row[2],
"format_version": row[3],
"rejected_reason": row[4],
"redactions": [],
}
# check for redactions
redactions_sql = (
"SELECT event_id, redacts FROM redactions WHERE redacts IN (%s)"
) % (",".join(["?"] * len(evs)), )
txn.execute(redactions_sql, evs)
for (redacter, redacted) in txn:
d = event_dict.get(redacted)
if d:
d["redactions"].append(redacter)
return event_dict
def _txn(txn):
for batch in batch_iter(server_name_and_key_ids, 50):
_get_keys(txn, batch)
return keys
def _txn(txn):
for batch in batch_iter(server_name_and_key_ids, 50):
_get_keys(txn, batch)
return keys