def setup_test_homeserver(self, *args, **kwargs): """ Set up the test homeserver, meant to be called by the overridable make_homeserver. It automatically passes through the test class's clock & reactor. Args: See tests.utils.setup_test_homeserver. Returns: synapse.server.HomeServer """ kwargs = dict(kwargs) kwargs.update(self._hs_args) if "config" not in kwargs: config = self.default_config() else: config = kwargs["config"] # Parse the config from a config dict into a HomeServerConfig config_obj = HomeServerConfig() config_obj.parse_config_dict(config, "", "") kwargs["config"] = config_obj hs = setup_test_homeserver(self.addCleanup, *args, **kwargs) stor = hs.get_datastore() # Run the database background updates, when running against "master". if hs.__class__.__name__ == "TestHomeServer": while not self.get_success( stor.db.updates.has_completed_background_updates() ): self.get_success(stor.db.updates.do_next_background_update(1)) return hs
def setUp(self): self.reactor = ThreadedMemoryReactorClock() self.mock_resolver = Mock() config_dict = default_config("test", parse=False) config_dict["federation_custom_ca_list"] = [get_test_ca_cert_file()] self._config = config = HomeServerConfig() config.parse_config_dict(config_dict, "", "") self.tls_factory = FederationPolicyForHTTPS(config) self.well_known_cache = TTLCache("test_cache", timer=self.reactor.seconds) self.had_well_known_cache = TTLCache("test_cache", timer=self.reactor.seconds) self.well_known_resolver = WellKnownResolver( self.reactor, Agent(self.reactor, contextFactory=self.tls_factory), b"test-agent", well_known_cache=self.well_known_cache, had_well_known_cache=self.had_well_known_cache, ) self.agent = MatrixFederationAgent( reactor=self.reactor, tls_client_options_factory=self.tls_factory, user_agent= "test-agent", # Note that this is unused since _well_known_resolver is provided. ip_blacklist=IPSet(), _srv_resolver=self.mock_resolver, _well_known_resolver=self.well_known_resolver, )
def setUp(self): self.reactor = ThreadedMemoryReactorClock() self.mock_resolver = Mock() config_dict = default_config("test", parse=False) config_dict["federation_custom_ca_list"] = [get_test_ca_cert_file()] self._config = config = HomeServerConfig() config.parse_config_dict(config_dict, "", "") self.tls_factory = ClientTLSOptionsFactory(config) self.well_known_cache = TTLCache("test_cache", timer=self.reactor.seconds) self.had_well_known_cache = TTLCache("test_cache", timer=self.reactor.seconds) self.well_known_resolver = WellKnownResolver( self.reactor, Agent(self.reactor, contextFactory=self.tls_factory), well_known_cache=self.well_known_cache, had_well_known_cache=self.had_well_known_cache, ) self.agent = MatrixFederationAgent( reactor=self.reactor, tls_client_options_factory=self.tls_factory, _srv_resolver=self.mock_resolver, _well_known_resolver=self.well_known_resolver, )
async def make_homeserver(reactor, config=None): """ Make a Homeserver suitable for running benchmarks against. Args: reactor: A Twisted reactor to run under. config: A HomeServerConfig to use, or None. """ cleanup_tasks = [] clock = Clock(reactor) if not config: config = default_config("test") config_obj = HomeServerConfig() config_obj.parse_config_dict(config, "", "") hs = await setup_test_homeserver(cleanup_tasks.append, config=config_obj, reactor=reactor, clock=clock) stor = hs.get_datastore() # Run the database background updates. if hasattr(stor.db.updates, "do_next_background_update"): while not await stor.db.updates.has_completed_background_updates(): await stor.db.updates.do_next_background_update(1) def cleanup(): for i in cleanup_tasks: i() return hs, clock.sleep, cleanup
def test_session_lifetime_must_not_be_exceeded_by_smaller_lifetimes(self): """ session_lifetime should logically be larger than, or at least as large as, all the different token lifetimes. Test that the user is faced with configuration errors if they make it smaller, as that configuration doesn't make sense. """ config_dict = default_config("test") # First test all the error conditions with self.assertRaises(ConfigError): HomeServerConfig().parse_config_dict({ "session_lifetime": "30m", "nonrefreshable_access_token_lifetime": "31m", **config_dict, }) with self.assertRaises(ConfigError): HomeServerConfig().parse_config_dict({ "session_lifetime": "30m", "refreshable_access_token_lifetime": "31m", **config_dict, }) with self.assertRaises(ConfigError): HomeServerConfig().parse_config_dict({ "session_lifetime": "30m", "refresh_token_lifetime": "31m", **config_dict, }) # Then test all the fine conditions HomeServerConfig().parse_config_dict({ "session_lifetime": "31m", "nonrefreshable_access_token_lifetime": "31m", **config_dict, }) HomeServerConfig().parse_config_dict({ "session_lifetime": "31m", "refreshable_access_token_lifetime": "31m", **config_dict, }) HomeServerConfig().parse_config_dict({ "session_lifetime": "31m", "refresh_token_lifetime": "31m", **config_dict })
def main() -> None: parser = argparse.ArgumentParser(description=( "Updates a synapse database to the latest schema and optionally runs background updates" " on it.")) parser.add_argument("-v", action="store_true") parser.add_argument( "--database-config", type=argparse.FileType("r"), required=True, help= "Synapse configuration file, giving the details of the database to be updated", ) parser.add_argument( "--run-background-updates", action="store_true", required=False, help="run background updates after upgrading the database schema", ) args = parser.parse_args() logging.basicConfig( level=logging.DEBUG if args.v else logging.INFO, format= "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(message)s", ) # Load, process and sanity-check the config. hs_config = yaml.safe_load(args.database_config) if "database" not in hs_config: sys.stderr.write( "The configuration file must have a 'database' section.\n") sys.exit(4) config = HomeServerConfig() config.parse_config_dict(hs_config, "", "") # Instantiate and initialise the homeserver object. hs = MockHomeserver(config) # Setup instantiates the store within the homeserver object and updates the # DB. hs.setup() if args.run_background_updates: run_background_updates(hs)
def test_parse_rc_federation(self): config_dict = default_config("test") config_dict["rc_federation"] = { "window_size": 20000, "sleep_limit": 693, "sleep_delay": 252, "reject_limit": 198, "concurrent": 7, } config = HomeServerConfig() config.parse_config_dict(config_dict, "", "") config_obj = config.rc_federation self.assertEqual(config_obj.window_size, 20000) self.assertEqual(config_obj.sleep_limit, 693) self.assertEqual(config_obj.sleep_delay, 252) self.assertEqual(config_obj.reject_limit, 198) self.assertEqual(config_obj.concurrent, 7)
def setUp(self): self.reactor = ThreadedMemoryReactorClock() self.mock_resolver = Mock() self.well_known_cache = TTLCache("test_cache", timer=self.reactor.seconds) config_dict = default_config("test", parse=False) config_dict["federation_custom_ca_list"] = [get_test_ca_cert_file()] # config_dict["trusted_key_servers"] = [] self._config = config = HomeServerConfig() config.parse_config_dict(config_dict) self.agent = MatrixFederationAgent( reactor=self.reactor, tls_client_options_factory=ClientTLSOptionsFactory(config), _well_known_tls_policy=TrustingTLSPolicyForHTTPS(), _srv_resolver=self.mock_resolver, _well_known_cache=self.well_known_cache, )
def setup_test_homeserver(self, *args: Any, **kwargs: Any) -> HomeServer: """ Set up the test homeserver, meant to be called by the overridable make_homeserver. It automatically passes through the test class's clock & reactor. Args: See tests.utils.setup_test_homeserver. Returns: synapse.server.HomeServer """ kwargs = dict(kwargs) kwargs.update(self._hs_args) if "config" not in kwargs: config = self.default_config() else: config = kwargs["config"] # Parse the config from a config dict into a HomeServerConfig config_obj = HomeServerConfig() config_obj.parse_config_dict(config, "", "") kwargs["config"] = config_obj async def run_bg_updates(): with LoggingContext("run_bg_updates"): self.get_success( stor.db_pool.updates.run_background_updates(False)) hs = setup_test_homeserver(self.addCleanup, *args, **kwargs) stor = hs.get_datastores().main # Run the database background updates, when running against "master". if hs.__class__.__name__ == "TestHomeServer": self.get_success(run_bg_updates()) return hs
def default_config(name): """ Create a reasonable test config. """ config_dict = { "server_name": name, "media_store_path": "media", "uploads_path": "uploads", # the test signing key is just an arbitrary ed25519 key to keep the config # parser happy "signing_key": "ed25519 a_lPym qvioDNmfExFBRPgdTU+wtFYKq4JfwFRv7sYVgWvmgJg", } config = HomeServerConfig() config.parse_config_dict(config_dict) # TODO: move this stuff into config_dict or get rid of it config.event_cache_size = 1 config.enable_registration = True config.enable_registration_captcha = False config.macaroon_secret_key = "not even a little secret" config.expire_access_token = False config.trusted_third_party_id_servers = [] config.room_invite_state_types = [] config.password_providers = [] config.worker_replication_url = "" config.worker_app = None config.email_enable_notifs = False config.block_non_admin_invites = False config.federation_domain_whitelist = None config.federation_rc_reject_limit = 10 config.federation_rc_sleep_limit = 10 config.federation_rc_sleep_delay = 100 config.federation_rc_concurrent = 10 config.filter_timeline_limit = 5000 config.user_directory_search_all_users = False config.user_consent_server_notice_content = None config.block_events_without_consent_error = None config.user_consent_at_registration = False config.user_consent_policy_name = "Privacy Policy" config.media_storage_providers = [] config.autocreate_auto_join_rooms = True config.auto_join_rooms = [] config.limit_usage_by_mau = False config.hs_disabled = False config.hs_disabled_message = "" config.hs_disabled_limit_type = "" config.max_mau_value = 50 config.mau_trial_days = 0 config.mau_stats_only = False config.mau_limits_reserved_threepids = [] config.admin_contact = None config.rc_messages_per_second = 10000 config.rc_message_burst_count = 10000 config.rc_registration.per_second = 10000 config.rc_registration.burst_count = 10000 config.rc_login_address.per_second = 10000 config.rc_login_address.burst_count = 10000 config.rc_login_account.per_second = 10000 config.rc_login_account.burst_count = 10000 config.rc_login_failed_attempts.per_second = 10000 config.rc_login_failed_attempts.burst_count = 10000 config.saml2_enabled = False config.public_baseurl = None config.default_identity_server = None config.key_refresh_interval = 24 * 60 * 60 * 1000 config.old_signing_keys = {} config.tls_fingerprints = [] config.use_frozen_dicts = False # we need a sane default_room_version, otherwise attempts to create rooms will # fail. config.default_room_version = "1" # disable user directory updates, because they get done in the # background, which upsets the test runner. config.update_user_directory = False return config
def default_config(name, parse=False): """ Create a reasonable test config. """ config_dict = { "server_name": name, "send_federation": False, "media_store_path": "media", # the test signing key is just an arbitrary ed25519 key to keep the config # parser happy "signing_key": "ed25519 a_lPym qvioDNmfExFBRPgdTU+wtFYKq4JfwFRv7sYVgWvmgJg", "event_cache_size": 1, "enable_registration": True, "enable_registration_captcha": False, "macaroon_secret_key": "not even a little secret", "password_providers": [], "worker_replication_url": "", "worker_app": None, "block_non_admin_invites": False, "federation_domain_whitelist": None, "filter_timeline_limit": 5000, "user_directory_search_all_users": False, "user_consent_server_notice_content": None, "block_events_without_consent_error": None, "user_consent_at_registration": False, "user_consent_policy_name": "Privacy Policy", "media_storage_providers": [], "autocreate_auto_join_rooms": True, "auto_join_rooms": [], "limit_usage_by_mau": False, "hs_disabled": False, "hs_disabled_message": "", "max_mau_value": 50, "mau_trial_days": 0, "mau_stats_only": False, "mau_limits_reserved_threepids": [], "admin_contact": None, "rc_message": { "per_second": 10000, "burst_count": 10000 }, "rc_registration": { "per_second": 10000, "burst_count": 10000 }, "rc_login": { "address": { "per_second": 10000, "burst_count": 10000 }, "account": { "per_second": 10000, "burst_count": 10000 }, "failed_attempts": { "per_second": 10000, "burst_count": 10000 }, }, "rc_joins": { "local": { "per_second": 10000, "burst_count": 10000 }, "remote": { "per_second": 10000, "burst_count": 10000 }, }, "rc_invites": { "per_room": { "per_second": 10000, "burst_count": 10000 }, "per_user": { "per_second": 10000, "burst_count": 10000 }, }, "rc_3pid_validation": { "per_second": 10000, "burst_count": 10000 }, "saml2_enabled": False, "public_baseurl": None, "default_identity_server": None, "key_refresh_interval": 24 * 60 * 60 * 1000, "old_signing_keys": {}, "tls_fingerprints": [], "use_frozen_dicts": False, # We need a sane default_room_version, otherwise attempts to create # rooms will fail. "default_room_version": DEFAULT_ROOM_VERSION, # disable user directory updates, because they get done in the # background, which upsets the test runner. "update_user_directory": False, "caches": { "global_factor": 1, "sync_response_cache_duration": 0 }, "listeners": [{ "port": 0, "type": "http" }], } if parse: config = HomeServerConfig() config.parse_config_dict(config_dict, "", "") return config return config_dict
def build_rc_config(settings: Optional[dict] = None): config_dict = default_config("test") config_dict.update(settings or {}) config = HomeServerConfig() config.parse_config_dict(config_dict, "", "") return config.ratelimiting.rc_federation
def build_rc_config(settings={}): config_dict = default_config("test") config_dict.update(settings) config = HomeServerConfig() config.parse_config_dict(config_dict, "", "") return config.rc_federation
def create_config(config_dir_path, data_dir_path, conf): """ Generates a config by interfacing with synapse's `synapse.config.homeserver.HomeSErverConfig`. See `synapse.config._base.Config.generate_config` for all options. Args: config_dir_path (str): Path to config directory. data_dir_path (str): Path to the data directory. conf (dict): Options for the config generator. example conf = { "server_name": "banterserver", "listeners": [ { "port": 8008, "resources": [{"names": ["client", "federation"]}], "tls": True, "type": "http", } ], "report_stats": True, "tls_certificate_path": self.tls_certificate_path, "tls_private_key_path": self.tls_private_key_path, } Returns (dict): {"filename": "config_content"} """ server_name = conf["server_name"] del conf["server_name"] server_config_in_use = conf.get("server_config_in_use") if server_config_in_use is not None: del conf["server_config_in_use"] database_conf = conf.get("database") if database_conf is not None: del conf["database"] if database_conf["name"] == "sqlite3": database_conf.setdefault( "args", {"database": join(data_dir_path, "homeserver.db")} ) config_args = { "config_dir_path": config_dir_path, "data_dir_path": data_dir_path, "server_name": server_name, "database_conf": database_conf, "generate_secrets": True, **conf, } home_server_config = HomeServerConfig() config_yaml = home_server_config.generate_config(**config_args) config = {} config.update(yaml.safe_load(config_yaml)) home_server_config.generate_missing_files(config, config_dir_path) return { "homeserver.yaml": config_yaml + "\n\nserver_config_in_use: {}".format(server_config_in_use) }
def main() -> None: parser = argparse.ArgumentParser() parser.add_argument( "--config-dir", default="CONFDIR", help= "The path where the config files are kept. Used to create filenames for " "things like the log config and the signing key. Default: %(default)s", ) parser.add_argument( "--data-dir", default="DATADIR", help= "The path where the data files are kept. Used to create filenames for " "things like the database and media store. Default: %(default)s", ) parser.add_argument( "--server-name", default="SERVERNAME", help= "The server name. Used to initialise the server_name config param, but also " "used in the names of some of the config files. Default: %(default)s", ) parser.add_argument( "--report-stats", action="store", help="Whether the generated config reports anonymized usage statistics", choices=["yes", "no"], ) parser.add_argument( "--generate-secrets", action="store_true", help= "Enable generation of new secrets for things like the macaroon_secret_key." "By default, these parameters will be left unset.", ) parser.add_argument( "-o", "--output-file", type=argparse.FileType("w"), default=sys.stdout, help="File to write the configuration to. Default: stdout", ) parser.add_argument( "--header-file", type=argparse.FileType("r"), help= "File from which to read a header, which will be printed before the " "generated config.", ) args = parser.parse_args() report_stats = args.report_stats if report_stats is not None: report_stats = report_stats == "yes" conf = HomeServerConfig().generate_config( config_dir_path=args.config_dir, data_dir_path=args.data_dir, server_name=args.server_name, generate_secrets=args.generate_secrets, report_stats=report_stats, ) if args.header_file: shutil.copyfileobj(args.header_file, args.output_file) args.output_file.write(conf)
def main() -> None: parser = argparse.ArgumentParser( description="A script to port an existing synapse SQLite database to" " a new PostgreSQL database.") parser.add_argument("-v", action="store_true") parser.add_argument( "--sqlite-database", required=True, help="The snapshot of the SQLite database file. This must not be" " currently used by a running synapse server", ) parser.add_argument( "--postgres-config", type=argparse.FileType("r"), required=True, help="The database config file for the PostgreSQL database", ) parser.add_argument("--curses", action="store_true", help="display a curses based progress UI") parser.add_argument( "--batch-size", type=int, default=1000, help="The number of rows to select from the SQLite table each" " iteration [default=1000]", ) args = parser.parse_args() logging.basicConfig( level=logging.DEBUG if args.v else logging.INFO, format= "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(message)s", filename="port-synapse.log" if args.curses else None, ) sqlite_config = { "name": "sqlite3", "args": { "database": args.sqlite_database, "cp_min": 1, "cp_max": 1, "check_same_thread": False, }, } hs_config = yaml.safe_load(args.postgres_config) if "database" not in hs_config: sys.stderr.write( "The configuration file must have a 'database' section.\n") sys.exit(4) postgres_config = hs_config["database"] if "name" not in postgres_config: sys.stderr.write("Malformed database config: no 'name'\n") sys.exit(2) if postgres_config["name"] != "psycopg2": sys.stderr.write("Database must use the 'psycopg2' connector.\n") sys.exit(3) config = HomeServerConfig() config.parse_config_dict(hs_config, "", "") def start(stdscr: Optional["curses.window"] = None) -> None: progress: Progress if stdscr: progress = CursesProgress(stdscr) else: progress = TerminalProgress() porter = Porter( sqlite_config=sqlite_config, progress=progress, batch_size=args.batch_size, hs_config=config, ) @defer.inlineCallbacks def run() -> Generator["defer.Deferred[Any]", Any, None]: with LoggingContext("synapse_port_db_run"): yield defer.ensureDeferred(porter.run()) reactor.callWhenRunning(run) reactor.run() if args.curses: curses.wrapper(start) else: start() if end_error: if end_error_exec_info: exc_type, exc_value, exc_traceback = end_error_exec_info traceback.print_exception(exc_type, exc_value, exc_traceback) sys.stderr.write(end_error) sys.exit(5)