def _connect(self, server_name: str, server_url: str) -> Optional[Tuple[str, GMatrixHttpApi]]: log.debug("Connecting", server=server_name) api = GMatrixHttpApi(server_url) username = self._username password = self._password if server_name != self._own_server_name: signer = make_signer() username = str(to_normalized_address(signer.address)) password = encode_hex(signer.sign(server_name.encode())) try: response = api.login( "m.login.password", user=username, password=password, device_id="room_ensurer" ) api.token = response["access_token"] except MatrixHttpLibError: log.warning("Could not connect to server", server_url=server_url) return None except MatrixRequestError: log.warning("Failed to login to server", server_url=server_url) return None log.debug("Connected", server=server_name) return server_name, api
def _connect(self, server_name: str, server_url: str) -> Tuple[str, GMatrixHttpApi]: log.debug("Connecting", server=server_name) api = GMatrixHttpApi(server_url) response = api.login("m.login.password", user=self._username, password=self._password) api.token = response["access_token"] log.debug("Connected", server=server_name) return server_name, api
def matrix_api_shell(address, password, server): am = AccountManager(os.path.expanduser("~/.ethereum/keystore")) signer = LocalSigner(am.get_privkey(to_checksum_address(address), password)) server_name = server.split("//")[1] matrix_password = encode_hex(signer.sign(server_name.encode())) api = GMatrixHttpApi(server) resp = api.login("m.login.password", user=to_normalized_address(address), password=matrix_password) api.token = resp["access_token"] IPython.embed( header=f"Use the `api` object to interact with matrix on {server}.")
def _connect(self, server_name: str, server_url: str) -> Tuple[str, GMatrixHttpApi]: log.debug("Connecting", server=server_name) api = GMatrixHttpApi(server_url) username = self._username password = self._password if server_name != self._own_server_name: signer = make_signer() username = str(to_normalized_address(signer.address)) password = encode_hex(signer.sign(server_name.encode())) response = api.login("m.login.password", user=username, password=password) api.token = response["access_token"] log.debug("Connected", server=server_name) return server_name, api
def purge( db_uri: str, server: str, credentials_file: TextIO, keep_newer: int, keep_min_msgs: int, parallel_purges: int, post_sql: TextIO, docker_restart_label: str, ) -> None: """ Purge historic data from rooms in a synapse server DB_URI: DB connection string: postgres://user:password@netloc:port/dbname SERVER: matrix synapse server url, e.g.: http://hostname All option can be passed through uppercase environment variables prefixed with 'MATRIX_' e.g.: export MATRIX_KEEP_MIN_MSGS=100 """ session = requests.Session() try: credentials = json.loads(credentials_file.read()) username = credentials["username"] password = credentials["password"] except (JSONDecodeError, UnicodeDecodeError, OSError, KeyError) as ex: click.secho(f"Invalid credentials file: {ex}", fg="red") sys.exit(1) api = GMatrixHttpApi(server) try: response = api.login("m.login.password", user=username, password=password) admin_access_token = response["access_token"] except (MatrixError, KeyError) as ex: click.secho(f"Could not log in to server {server}: {ex}") sys.exit(1) try: with psycopg2.connect(db_uri) as db, db.cursor() as cur: purges: Dict[str, str] = dict() def wait_and_purge_room(room_id: str = None, event_id: str = None) -> None: """ Wait for available slots in parallel_purges and purge room If room_id is None, just wait for current purges to complete and return If event_id is None, purge all events in room """ while len(purges) >= (parallel_purges if room_id else 1): # wait and clear completed purges time.sleep(1) for _room_id, purge_id in list(purges.items()): response = session.get( urljoin( server, "/_matrix/client/r0/admin/purge_history_status/" + quote(purge_id), ), params={"access_token": admin_access_token}, ) assert response.status_code == 200, f"{response!r} => {response.text!r}" if response.json()["status"] != "active": click.secho( f"Finished purge: room {_room_id!r}, purge {purge_id!r}" ) purges.pop(_room_id) if not room_id: return body: Dict[str, Any] = {"delete_local_events": True} if event_id: body["purge_up_to_event_id"] = event_id else: body["purge_up_to_ts"] = int(time.time() * 1000) response = session.post( urljoin( server, "/_matrix/client/r0/admin/purge_history/" + quote(room_id)), params={"access_token": admin_access_token}, json=body, ) if response.status_code == 200: purge_id = response.json()["purge_id"] purges[room_id] = purge_id return if not keep_newer and not keep_min_msgs: click.confirm( "No --keep-newer nor --keep-min-msgs option provided. Purge all history?", abort=True, ) ts_ms = None if keep_newer: ts = datetime.datetime.now() - datetime.timedelta(keep_newer) ts_ms = int(ts.timestamp() * 1000) cur.execute("SELECT room_id FROM rooms ;") all_rooms = {row for row, in cur} click.secho(f"Processing {len(all_rooms)} rooms") for room_id in all_rooms: # no --keep-min-msgs nor --keep-newer, purge everything if not keep_newer and not keep_min_msgs: wait_and_purge_room(room_id) continue cur.execute( f""" SELECT event_id FROM ( SELECT event_id, received_ts, COUNT(*) OVER (ORDER BY received_ts DESC) AS msg_count_above FROM events WHERE room_id=%(room_id)s AND type='m.room.message' ORDER BY received_ts DESC ) t WHERE true {'AND received_ts < %(ts_ms)s' if keep_newer else ''} {'AND msg_count_above > %(keep_min_msgs)s' if keep_min_msgs else ''} LIMIT 1 ;""", { "room_id": room_id, "ts_ms": ts_ms, "keep_min_msgs": keep_min_msgs }, ) if cur.rowcount: event_id, = cur.fetchone() wait_and_purge_room(room_id, event_id) # else: room doesn't have messages eligible for purging, skip wait_and_purge_room(None) if post_sql: click.secho(f"Running {post_sql.name!r}") with psycopg2.connect(db_uri) as db, db.cursor() as cur: cur.execute(post_sql.read()) click.secho(f"Results {cur.rowcount}:") for i, row in enumerate(cur): click.secho(f"{i}: {row}") finally: if docker_restart_label: client = docker.from_env() for container in client.containers.list(): if container.attrs["State"][ "Status"] != "running" or not container.attrs[ "Config"]["Labels"].get(docker_restart_label): continue try: # parse container's env vars env_vars: Dict[str, Any] = dict( itemgetter(0, 2)(e.partition("=")) for e in container.attrs["Config"]["Env"]) remote_config_file = ( env_vars.get("URL_KNOWN_FEDERATION_SERVERS") or URL_KNOWN_FEDERATION_SERVERS_DEFAULT) # fetch remote file remote_whitelist = yaml.load( requests.get(remote_config_file).text) # fetch local list from container's synapse config local_whitelist = yaml.load( container.exec_run([ "cat", SYNAPSE_CONFIG_PATH ]).output)["federation_domain_whitelist"] # if list didn't change, don't proceed to restart container if local_whitelist and remote_whitelist == local_whitelist: continue click.secho( f"Whitelist changed. Restarting. new_list={remote_whitelist!r}" ) except ( KeyError, IndexError, requests.RequestException, yaml.scanner.ScannerError, ) as ex: click.secho( f"An error ocurred while fetching whitelists: {ex!r}\n" "Restarting anyway", err=True, ) # restart container container.restart(timeout=30)
def purge(server: str, credentials_file: TextIO, docker_restart_label: str) -> None: """ Purge inactive users from broadcast rooms SERVER: matrix synapse server url, e.g.: http://hostname All option can be passed through uppercase environment variables prefixed with 'MATRIX_' """ try: credentials = json.loads(credentials_file.read()) username = credentials["username"] password = credentials["password"] except (JSONDecodeError, UnicodeDecodeError, OSError, KeyError) as ex: click.secho(f"Invalid credentials file: {ex}", fg="red") sys.exit(1) api = GMatrixHttpApi(server) try: api.login("m.login.password", user=username, password=password) except (MatrixError, KeyError) as ex: click.secho(f"Could not log in to server {server}: {ex}") sys.exit(1) try: global_user_activity = { "last_update": int(time.time()) - USER_PURGING_THRESHOLD - 1, "network_to_users": {}, } try: global_user_activity = json.loads(USER_ACTIVITY_PATH.read_text()) except JSONDecodeError: click.secho( f"{USER_ACTIVITY_PATH} is not a valid JSON. Starting with empty list" ) except FileNotFoundError: click.secho( f"{USER_ACTIVITY_PATH} not found. Starting with empty list") # check if there are new networks to add for network in Networks: if str(network.value) in global_user_activity["network_to_users"]: continue global_user_activity["network_to_users"][str( network.value)] = dict() # get broadcast room ids for all networks network_to_broadcast_rooms = get_network_to_broadcast_rooms(api) new_global_user_activity = run_user_purger(api, global_user_activity, network_to_broadcast_rooms) # write the updated user activity to file USER_ACTIVITY_PATH.write_text(json.dumps(new_global_user_activity)) finally: if docker_restart_label: client = docker.from_env() # type: ignore for container in client.containers.list(): if container.attrs["State"][ "Status"] != "running" or not container.attrs[ "Config"]["Labels"].get(docker_restart_label): continue try: # parse container's env vars env_vars: Dict[str, Any] = dict( itemgetter(0, 2)(e.partition("=")) for e in container.attrs["Config"]["Env"]) remote_config_file = ( env_vars.get("URL_KNOWN_FEDERATION_SERVERS") or URL_KNOWN_FEDERATION_SERVERS_DEFAULT) # fetch remote file remote_whitelist = yaml.load( requests.get(remote_config_file).text) # fetch local list from container's synapse config local_whitelist = yaml.load( container.exec_run([ "cat", SYNAPSE_CONFIG_PATH ]).output)["federation_domain_whitelist"] # if list didn't change, don't proceed to restart container if local_whitelist and remote_whitelist == local_whitelist: continue click.secho( f"Whitelist changed. Restarting. new_list={remote_whitelist!r}" ) except ( KeyError, IndexError, requests.RequestException, yaml.scanner.ScannerError, ) as ex: click.secho( f"An error ocurred while fetching whitelists: {ex!r}\n" "Restarting anyway", err=True, ) # restart container container.restart(timeout=30)
def purge( server: str, credentials_file: TextIO, docker_restart_label: Optional[str], url_known_federation_servers: str, ) -> None: """ Purge inactive users from broadcast rooms SERVER: matrix synapse server url, e.g.: http://hostname All option can be passed through uppercase environment variables prefixed with 'MATRIX_' """ try: credentials = json.loads(credentials_file.read()) username = credentials["username"] password = credentials["password"] except (JSONDecodeError, UnicodeDecodeError, OSError, KeyError) as ex: click.secho(f"Invalid credentials file: {ex}", fg="red") sys.exit(1) api = GMatrixHttpApi(server) try: response = api.login("m.login.password", user=username, password=password, device_id="purger") api.token = response["access_token"] except (MatrixError, KeyError) as ex: click.secho(f"Could not log in to server {server}: {ex}") sys.exit(1) try: global_user_activity: UserActivityInfo = { "last_update": int(time.time()) - USER_PURGING_THRESHOLD - 1, "network_to_users": {}, } try: global_user_activity = json.loads(USER_ACTIVITY_PATH.read_text()) except JSONDecodeError: click.secho( f"{USER_ACTIVITY_PATH} is not a valid JSON. Starting with empty list" ) except FileNotFoundError: click.secho( f"{USER_ACTIVITY_PATH} not found. Starting with empty list") # check if there are new networks to add for network in Networks: if str(network.value) in global_user_activity["network_to_users"]: continue global_user_activity["network_to_users"][str( network.value)] = dict() new_global_user_activity = run_user_purger(api, global_user_activity) # write the updated user activity to file USER_ACTIVITY_PATH.write_text( json.dumps(cast(Dict[str, Any], new_global_user_activity))) finally: if docker_restart_label: if not url_known_federation_servers: # In case an empty env var is set url_known_federation_servers = DEFAULT_MATRIX_KNOWN_SERVERS[ Environment.PRODUCTION] # fetch remote whiltelist try: remote_whitelist = json.loads( requests.get( url_known_federation_servers).text)["all_servers"] except (requests.RequestException, JSONDecodeError, KeyError) as ex: click.secho( f"Error while fetching whitelist: {ex!r}. " f"Ignoring, containers will be restarted.", err=True, ) # An empty whitelist will cause the container to be restarted remote_whitelist = [] client = docker.from_env() # pylint: disable=no-member for container in client.containers.list(): if container.attrs["State"][ "Status"] != "running" or not container.attrs[ "Config"]["Labels"].get(docker_restart_label): continue try: # fetch local list from container's synapse config local_whitelist = yaml.safe_load( container.exec_run([ "cat", SYNAPSE_CONFIG_PATH ]).output)["federation_domain_whitelist"] # if list didn't change, don't proceed to restart container if local_whitelist and remote_whitelist == local_whitelist: continue click.secho( f"Whitelist changed. Restarting. new_list={remote_whitelist!r}" ) except (KeyError, IndexError) as ex: click.secho( f"Error fetching container status: {ex!r}. Restarting anyway.", err=True, ) # restart container container.restart(timeout=30)