def __init__(self, main_store_class, hs): # Note we pass in the main store class here as workers use a different main # store. self.databases = [] main = None state = None persist_events = None for database_config in hs.config.database.databases: db_name = database_config.name engine = create_engine(database_config.config) with make_conn(database_config, engine, "startup") as db_conn: logger.info("[database config %r]: Checking database server", db_name) engine.check_database(db_conn) logger.info( "[database config %r]: Preparing for databases %r", db_name, database_config.databases, ) prepare_database( db_conn, engine, hs.config, databases=database_config.databases, ) database = DatabasePool(hs, database_config, engine) if "main" in database_config.databases: logger.info( "[database config %r]: Starting 'main' database", db_name) # Sanity check we don't try and configure the main store on # multiple databases. if main: raise Exception("'main' data store already configured") main = main_store_class(database, db_conn, hs) # If we're on a process that can persist events also # instantiate a `PersistEventsStore` if hs.get_instance_name( ) in hs.config.worker.writers.events: persist_events = PersistEventsStore(hs, database, main) if "state" in database_config.databases: logger.info( "[database config %r]: Starting 'state' database", db_name) # Sanity check we don't try and configure the state store on # multiple databases. if state: raise Exception( "'state' data store already configured") state = StateGroupDataStore(database, db_conn, hs) db_conn.commit() self.databases.append(database) logger.info("[database config %r]: prepared", db_name) # Closing the context manager doesn't close the connection. # psycopg will close the connection when the object gets GCed, but *only* # if the PID is the same as when the connection was opened [1], and # it may not be if we fork in the meantime. # # [1]: https://github.com/psycopg/psycopg2/blob/2_8_5/psycopg/connection_type.c#L1378 db_conn.close() # Sanity check that we have actually configured all the required stores. if not main: raise Exception("No 'main' database configured") if not state: raise Exception("No 'state' database configured") # We use local variables here to ensure that the databases do not have # optional types. self.main = main self.state = state self.persist_events = persist_events
def _calculate_chain_cover_txn( self, txn: Cursor, last_room_id: str, last_depth: int, last_stream: int, batch_size: Optional[int], single_room: bool, ) -> _CalculateChainCover: """Calculate the chain cover for `batch_size` events, ordered by `(room_id, depth, stream)`. Args: txn, last_room_id, last_depth, last_stream: The `(room_id, depth, stream)` tuple to fetch results after. batch_size: The maximum number of events to process. If None then no limit. single_room: Whether to calculate the index for just the given room. """ # Get the next set of events in the room (that we haven't already # computed chain cover for). We do this in topological order. # We want to do a `(topological_ordering, stream_ordering) > (?,?)` # comparison, but that is not supported on older SQLite versions tuple_clause, tuple_args = make_tuple_comparison_clause([ ("events.room_id", last_room_id), ("topological_ordering", last_depth), ("stream_ordering", last_stream), ], ) extra_clause = "" if single_room: extra_clause = "AND events.room_id = ?" tuple_args.append(last_room_id) sql = """ SELECT event_id, state_events.type, state_events.state_key, topological_ordering, stream_ordering, events.room_id FROM events INNER JOIN state_events USING (event_id) LEFT JOIN event_auth_chains USING (event_id) LEFT JOIN event_auth_chain_to_calculate USING (event_id) WHERE event_auth_chains.event_id IS NULL AND event_auth_chain_to_calculate.event_id IS NULL AND %(tuple_cmp)s %(extra)s ORDER BY events.room_id, topological_ordering, stream_ordering %(limit)s """ % { "tuple_cmp": tuple_clause, "limit": "LIMIT ?" if batch_size is not None else "", "extra": extra_clause, } if batch_size is not None: tuple_args.append(batch_size) txn.execute(sql, tuple_args) rows = txn.fetchall() # Put the results in the necessary format for # `_add_chain_cover_index` event_to_room_id = {row[0]: row[5] for row in rows} event_to_types = {row[0]: (row[1], row[2]) for row in rows} # Calculate the new last position we've processed up to. new_last_depth: int = rows[-1][3] if rows else last_depth new_last_stream: int = rows[-1][4] if rows else last_stream new_last_room_id: str = rows[-1][5] if rows else "" # Map from room_id to last depth/stream_ordering processed for the room, # excluding the last room (which we're likely still processing). We also # need to include the room passed in if it's not included in the result # set (as we then know we've processed all events in said room). # # This is the set of rooms that we can now safely flip the # `has_auth_chain_index` bit for. finished_rooms = { row[5]: (row[3], row[4]) for row in rows if row[5] != new_last_room_id } if last_room_id not in finished_rooms and last_room_id != new_last_room_id: finished_rooms[last_room_id] = (last_depth, last_stream) count = len(rows) # We also need to fetch the auth events for them. auth_events = self.db_pool.simple_select_many_txn( txn, table="event_auth", column="event_id", iterable=event_to_room_id, keyvalues={}, retcols=("event_id", "auth_id"), ) event_to_auth_chain: Dict[str, List[str]] = {} for row in auth_events: event_to_auth_chain.setdefault(row["event_id"], []).append(row["auth_id"]) # Calculate and persist the chain cover index for this set of events. # # Annoyingly we need to gut wrench into the persit event store so that # we can reuse the function to calculate the chain cover for rooms. PersistEventsStore._add_chain_cover_index( txn, self.db_pool, self.event_chain_id_gen, event_to_room_id, event_to_types, event_to_auth_chain, ) return _CalculateChainCover( room_id=new_last_room_id, depth=new_last_depth, stream=new_last_stream, processed_count=count, finished_room_map=finished_rooms, )
def __init__(self, main_store_class, hs): # Note we pass in the main store class here as workers use a different main # store. self.databases = [] main = None state = None persist_events = None for database_config in hs.config.database.databases: db_name = database_config.name engine = create_engine(database_config.config) with make_conn(database_config, engine) as db_conn: logger.info("Preparing database %r...", db_name) engine.check_database(db_conn) prepare_database( db_conn, engine, hs.config, databases=database_config.databases, ) database = DatabasePool(hs, database_config, engine) if "main" in database_config.databases: logger.info("Starting 'main' data store") # Sanity check we don't try and configure the main store on # multiple databases. if main: raise Exception("'main' data store already configured") main = main_store_class(database, db_conn, hs) # If we're on a process that can persist events also # instantiate a `PersistEventsStore` if hs.config.worker.writers.events == hs.get_instance_name( ): persist_events = PersistEventsStore(hs, database, main) if "state" in database_config.databases: logger.info("Starting 'state' data store") # Sanity check we don't try and configure the state store on # multiple databases. if state: raise Exception( "'state' data store already configured") state = StateGroupDataStore(database, db_conn, hs) db_conn.commit() self.databases.append(database) logger.info("Database %r prepared", db_name) # Sanity check that we have actually configured all the required stores. if not main: raise Exception("No 'main' data store configured") if not state: raise Exception("No 'main' data store configured") # We use local variables here to ensure that the databases do not have # optional types. self.main = main self.state = state self.persist_events = persist_events