def setup_test_homeserver(self, *args, **kwargs):
"""
Set up the test homeserver, meant to be called by the overridable
make_homeserver. It automatically passes through the test class's
clock & reactor.
Args:
See tests.utils.setup_test_homeserver.
Returns:
synapse.server.HomeServer
"""
kwargs = dict(kwargs)
kwargs.update(self._hs_args)
if "config" not in kwargs:
config = self.default_config()
else:
config = kwargs["config"]
# Parse the config from a config dict into a HomeServerConfig
config_obj = HomeServerConfig()
config_obj.parse_config_dict(config, "", "")
kwargs["config"] = config_obj
hs = setup_test_homeserver(self.addCleanup, *args, **kwargs)
stor = hs.get_datastore()
# Run the database background updates, when running against "master".
if hs.__class__.__name__ == "TestHomeServer":
while not self.get_success(
stor.db.updates.has_completed_background_updates()
):
self.get_success(stor.db.updates.do_next_background_update(1))
return hs
def setUp(self):
self.reactor = ThreadedMemoryReactorClock()
self.mock_resolver = Mock()
config_dict = default_config("test", parse=False)
config_dict["federation_custom_ca_list"] = [get_test_ca_cert_file()]
self._config = config = HomeServerConfig()
config.parse_config_dict(config_dict, "", "")
self.tls_factory = FederationPolicyForHTTPS(config)
self.well_known_cache = TTLCache("test_cache",
timer=self.reactor.seconds)
self.had_well_known_cache = TTLCache("test_cache",
timer=self.reactor.seconds)
self.well_known_resolver = WellKnownResolver(
self.reactor,
Agent(self.reactor, contextFactory=self.tls_factory),
b"test-agent",
well_known_cache=self.well_known_cache,
had_well_known_cache=self.had_well_known_cache,
)
self.agent = MatrixFederationAgent(
reactor=self.reactor,
tls_client_options_factory=self.tls_factory,
user_agent=
"test-agent", # Note that this is unused since _well_known_resolver is provided.
ip_blacklist=IPSet(),
_srv_resolver=self.mock_resolver,
_well_known_resolver=self.well_known_resolver,
)
def setUp(self):
self.reactor = ThreadedMemoryReactorClock()
self.mock_resolver = Mock()
config_dict = default_config("test", parse=False)
config_dict["federation_custom_ca_list"] = [get_test_ca_cert_file()]
self._config = config = HomeServerConfig()
config.parse_config_dict(config_dict, "", "")
self.tls_factory = ClientTLSOptionsFactory(config)
self.well_known_cache = TTLCache("test_cache",
timer=self.reactor.seconds)
self.had_well_known_cache = TTLCache("test_cache",
timer=self.reactor.seconds)
self.well_known_resolver = WellKnownResolver(
self.reactor,
Agent(self.reactor, contextFactory=self.tls_factory),
well_known_cache=self.well_known_cache,
had_well_known_cache=self.had_well_known_cache,
)
self.agent = MatrixFederationAgent(
reactor=self.reactor,
tls_client_options_factory=self.tls_factory,
_srv_resolver=self.mock_resolver,
_well_known_resolver=self.well_known_resolver,
)
async def make_homeserver(reactor, config=None):
"""
Make a Homeserver suitable for running benchmarks against.
Args:
reactor: A Twisted reactor to run under.
config: A HomeServerConfig to use, or None.
"""
cleanup_tasks = []
clock = Clock(reactor)
if not config:
config = default_config("test")
config_obj = HomeServerConfig()
config_obj.parse_config_dict(config, "", "")
hs = await setup_test_homeserver(cleanup_tasks.append,
config=config_obj,
reactor=reactor,
clock=clock)
stor = hs.get_datastore()
# Run the database background updates.
if hasattr(stor.db.updates, "do_next_background_update"):
while not await stor.db.updates.has_completed_background_updates():
await stor.db.updates.do_next_background_update(1)
def cleanup():
for i in cleanup_tasks:
i()
return hs, clock.sleep, cleanup
def test_session_lifetime_must_not_be_exceeded_by_smaller_lifetimes(self):
"""
session_lifetime should logically be larger than, or at least as large as,
all the different token lifetimes.
Test that the user is faced with configuration errors if they make it
smaller, as that configuration doesn't make sense.
"""
config_dict = default_config("test")
# First test all the error conditions
with self.assertRaises(ConfigError):
HomeServerConfig().parse_config_dict({
"session_lifetime":
"30m",
"nonrefreshable_access_token_lifetime":
"31m",
**config_dict,
})
with self.assertRaises(ConfigError):
HomeServerConfig().parse_config_dict({
"session_lifetime":
"30m",
"refreshable_access_token_lifetime":
"31m",
**config_dict,
})
with self.assertRaises(ConfigError):
HomeServerConfig().parse_config_dict({
"session_lifetime": "30m",
"refresh_token_lifetime": "31m",
**config_dict,
})
# Then test all the fine conditions
HomeServerConfig().parse_config_dict({
"session_lifetime":
"31m",
"nonrefreshable_access_token_lifetime":
"31m",
**config_dict,
})
HomeServerConfig().parse_config_dict({
"session_lifetime":
"31m",
"refreshable_access_token_lifetime":
"31m",
**config_dict,
})
HomeServerConfig().parse_config_dict({
"session_lifetime": "31m",
"refresh_token_lifetime": "31m",
**config_dict
})
def main() -> None:
parser = argparse.ArgumentParser(description=(
"Updates a synapse database to the latest schema and optionally runs background updates"
" on it."))
parser.add_argument("-v", action="store_true")
parser.add_argument(
"--database-config",
type=argparse.FileType("r"),
required=True,
help=
"Synapse configuration file, giving the details of the database to be updated",
)
parser.add_argument(
"--run-background-updates",
action="store_true",
required=False,
help="run background updates after upgrading the database schema",
)
args = parser.parse_args()
logging.basicConfig(
level=logging.DEBUG if args.v else logging.INFO,
format=
"%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(message)s",
)
# Load, process and sanity-check the config.
hs_config = yaml.safe_load(args.database_config)
if "database" not in hs_config:
sys.stderr.write(
"The configuration file must have a 'database' section.\n")
sys.exit(4)
config = HomeServerConfig()
config.parse_config_dict(hs_config, "", "")
# Instantiate and initialise the homeserver object.
hs = MockHomeserver(config)
# Setup instantiates the store within the homeserver object and updates the
# DB.
hs.setup()
if args.run_background_updates:
run_background_updates(hs)
def test_parse_rc_federation(self):
config_dict = default_config("test")
config_dict["rc_federation"] = {
"window_size": 20000,
"sleep_limit": 693,
"sleep_delay": 252,
"reject_limit": 198,
"concurrent": 7,
}
config = HomeServerConfig()
config.parse_config_dict(config_dict, "", "")
config_obj = config.rc_federation
self.assertEqual(config_obj.window_size, 20000)
self.assertEqual(config_obj.sleep_limit, 693)
self.assertEqual(config_obj.sleep_delay, 252)
self.assertEqual(config_obj.reject_limit, 198)
self.assertEqual(config_obj.concurrent, 7)
def setUp(self):
self.reactor = ThreadedMemoryReactorClock()
self.mock_resolver = Mock()
self.well_known_cache = TTLCache("test_cache",
timer=self.reactor.seconds)
config_dict = default_config("test", parse=False)
config_dict["federation_custom_ca_list"] = [get_test_ca_cert_file()]
# config_dict["trusted_key_servers"] = []
self._config = config = HomeServerConfig()
config.parse_config_dict(config_dict)
self.agent = MatrixFederationAgent(
reactor=self.reactor,
tls_client_options_factory=ClientTLSOptionsFactory(config),
_well_known_tls_policy=TrustingTLSPolicyForHTTPS(),
_srv_resolver=self.mock_resolver,
_well_known_cache=self.well_known_cache,
)
def setup_test_homeserver(self, *args: Any, **kwargs: Any) -> HomeServer:
"""
Set up the test homeserver, meant to be called by the overridable
make_homeserver. It automatically passes through the test class's
clock & reactor.
Args:
See tests.utils.setup_test_homeserver.
Returns:
synapse.server.HomeServer
"""
kwargs = dict(kwargs)
kwargs.update(self._hs_args)
if "config" not in kwargs:
config = self.default_config()
else:
config = kwargs["config"]
# Parse the config from a config dict into a HomeServerConfig
config_obj = HomeServerConfig()
config_obj.parse_config_dict(config, "", "")
kwargs["config"] = config_obj
async def run_bg_updates():
with LoggingContext("run_bg_updates"):
self.get_success(
stor.db_pool.updates.run_background_updates(False))
hs = setup_test_homeserver(self.addCleanup, *args, **kwargs)
stor = hs.get_datastores().main
# Run the database background updates, when running against "master".
if hs.__class__.__name__ == "TestHomeServer":
self.get_success(run_bg_updates())
return hs
def default_config(name):
"""
Create a reasonable test config.
"""
config_dict = {
"server_name":
name,
"media_store_path":
"media",
"uploads_path":
"uploads",
# the test signing key is just an arbitrary ed25519 key to keep the config
# parser happy
"signing_key":
"ed25519 a_lPym qvioDNmfExFBRPgdTU+wtFYKq4JfwFRv7sYVgWvmgJg",
}
config = HomeServerConfig()
config.parse_config_dict(config_dict)
# TODO: move this stuff into config_dict or get rid of it
config.event_cache_size = 1
config.enable_registration = True
config.enable_registration_captcha = False
config.macaroon_secret_key = "not even a little secret"
config.expire_access_token = False
config.trusted_third_party_id_servers = []
config.room_invite_state_types = []
config.password_providers = []
config.worker_replication_url = ""
config.worker_app = None
config.email_enable_notifs = False
config.block_non_admin_invites = False
config.federation_domain_whitelist = None
config.federation_rc_reject_limit = 10
config.federation_rc_sleep_limit = 10
config.federation_rc_sleep_delay = 100
config.federation_rc_concurrent = 10
config.filter_timeline_limit = 5000
config.user_directory_search_all_users = False
config.user_consent_server_notice_content = None
config.block_events_without_consent_error = None
config.user_consent_at_registration = False
config.user_consent_policy_name = "Privacy Policy"
config.media_storage_providers = []
config.autocreate_auto_join_rooms = True
config.auto_join_rooms = []
config.limit_usage_by_mau = False
config.hs_disabled = False
config.hs_disabled_message = ""
config.hs_disabled_limit_type = ""
config.max_mau_value = 50
config.mau_trial_days = 0
config.mau_stats_only = False
config.mau_limits_reserved_threepids = []
config.admin_contact = None
config.rc_messages_per_second = 10000
config.rc_message_burst_count = 10000
config.rc_registration.per_second = 10000
config.rc_registration.burst_count = 10000
config.rc_login_address.per_second = 10000
config.rc_login_address.burst_count = 10000
config.rc_login_account.per_second = 10000
config.rc_login_account.burst_count = 10000
config.rc_login_failed_attempts.per_second = 10000
config.rc_login_failed_attempts.burst_count = 10000
config.saml2_enabled = False
config.public_baseurl = None
config.default_identity_server = None
config.key_refresh_interval = 24 * 60 * 60 * 1000
config.old_signing_keys = {}
config.tls_fingerprints = []
config.use_frozen_dicts = False
# we need a sane default_room_version, otherwise attempts to create rooms will
# fail.
config.default_room_version = "1"
# disable user directory updates, because they get done in the
# background, which upsets the test runner.
config.update_user_directory = False
return config
def default_config(name, parse=False):
"""
Create a reasonable test config.
"""
config_dict = {
"server_name": name,
"send_federation": False,
"media_store_path": "media",
# the test signing key is just an arbitrary ed25519 key to keep the config
# parser happy
"signing_key":
"ed25519 a_lPym qvioDNmfExFBRPgdTU+wtFYKq4JfwFRv7sYVgWvmgJg",
"event_cache_size": 1,
"enable_registration": True,
"enable_registration_captcha": False,
"macaroon_secret_key": "not even a little secret",
"password_providers": [],
"worker_replication_url": "",
"worker_app": None,
"block_non_admin_invites": False,
"federation_domain_whitelist": None,
"filter_timeline_limit": 5000,
"user_directory_search_all_users": False,
"user_consent_server_notice_content": None,
"block_events_without_consent_error": None,
"user_consent_at_registration": False,
"user_consent_policy_name": "Privacy Policy",
"media_storage_providers": [],
"autocreate_auto_join_rooms": True,
"auto_join_rooms": [],
"limit_usage_by_mau": False,
"hs_disabled": False,
"hs_disabled_message": "",
"max_mau_value": 50,
"mau_trial_days": 0,
"mau_stats_only": False,
"mau_limits_reserved_threepids": [],
"admin_contact": None,
"rc_message": {
"per_second": 10000,
"burst_count": 10000
},
"rc_registration": {
"per_second": 10000,
"burst_count": 10000
},
"rc_login": {
"address": {
"per_second": 10000,
"burst_count": 10000
},
"account": {
"per_second": 10000,
"burst_count": 10000
},
"failed_attempts": {
"per_second": 10000,
"burst_count": 10000
},
},
"rc_joins": {
"local": {
"per_second": 10000,
"burst_count": 10000
},
"remote": {
"per_second": 10000,
"burst_count": 10000
},
},
"rc_invites": {
"per_room": {
"per_second": 10000,
"burst_count": 10000
},
"per_user": {
"per_second": 10000,
"burst_count": 10000
},
},
"rc_3pid_validation": {
"per_second": 10000,
"burst_count": 10000
},
"saml2_enabled": False,
"public_baseurl": None,
"default_identity_server": None,
"key_refresh_interval": 24 * 60 * 60 * 1000,
"old_signing_keys": {},
"tls_fingerprints": [],
"use_frozen_dicts": False,
# We need a sane default_room_version, otherwise attempts to create
# rooms will fail.
"default_room_version": DEFAULT_ROOM_VERSION,
# disable user directory updates, because they get done in the
# background, which upsets the test runner.
"update_user_directory": False,
"caches": {
"global_factor": 1,
"sync_response_cache_duration": 0
},
"listeners": [{
"port": 0,
"type": "http"
}],
}
if parse:
config = HomeServerConfig()
config.parse_config_dict(config_dict, "", "")
return config
return config_dict
def build_rc_config(settings: Optional[dict] = None):
config_dict = default_config("test")
config_dict.update(settings or {})
config = HomeServerConfig()
config.parse_config_dict(config_dict, "", "")
return config.ratelimiting.rc_federation
def build_rc_config(settings={}):
config_dict = default_config("test")
config_dict.update(settings)
config = HomeServerConfig()
config.parse_config_dict(config_dict, "", "")
return config.rc_federation
def create_config(config_dir_path, data_dir_path, conf):
"""
Generates a config by interfacing with synapse's
`synapse.config.homeserver.HomeSErverConfig`. See
`synapse.config._base.Config.generate_config` for all options.
Args:
config_dir_path (str): Path to config directory.
data_dir_path (str): Path to the data directory.
conf (dict): Options for the config generator.
example conf = {
"server_name": "banterserver",
"listeners": [
{
"port": 8008,
"resources": [{"names": ["client", "federation"]}],
"tls": True,
"type": "http",
}
],
"report_stats": True,
"tls_certificate_path": self.tls_certificate_path,
"tls_private_key_path": self.tls_private_key_path,
}
Returns
(dict): {"filename": "config_content"}
"""
server_name = conf["server_name"]
del conf["server_name"]
server_config_in_use = conf.get("server_config_in_use")
if server_config_in_use is not None:
del conf["server_config_in_use"]
database_conf = conf.get("database")
if database_conf is not None:
del conf["database"]
if database_conf["name"] == "sqlite3":
database_conf.setdefault(
"args", {"database": join(data_dir_path, "homeserver.db")}
)
config_args = {
"config_dir_path": config_dir_path,
"data_dir_path": data_dir_path,
"server_name": server_name,
"database_conf": database_conf,
"generate_secrets": True,
**conf,
}
home_server_config = HomeServerConfig()
config_yaml = home_server_config.generate_config(**config_args)
config = {}
config.update(yaml.safe_load(config_yaml))
home_server_config.generate_missing_files(config, config_dir_path)
return {
"homeserver.yaml": config_yaml
+ "\n\nserver_config_in_use: {}".format(server_config_in_use)
}
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument(
"--config-dir",
default="CONFDIR",
help=
"The path where the config files are kept. Used to create filenames for "
"things like the log config and the signing key. Default: %(default)s",
)
parser.add_argument(
"--data-dir",
default="DATADIR",
help=
"The path where the data files are kept. Used to create filenames for "
"things like the database and media store. Default: %(default)s",
)
parser.add_argument(
"--server-name",
default="SERVERNAME",
help=
"The server name. Used to initialise the server_name config param, but also "
"used in the names of some of the config files. Default: %(default)s",
)
parser.add_argument(
"--report-stats",
action="store",
help="Whether the generated config reports anonymized usage statistics",
choices=["yes", "no"],
)
parser.add_argument(
"--generate-secrets",
action="store_true",
help=
"Enable generation of new secrets for things like the macaroon_secret_key."
"By default, these parameters will be left unset.",
)
parser.add_argument(
"-o",
"--output-file",
type=argparse.FileType("w"),
default=sys.stdout,
help="File to write the configuration to. Default: stdout",
)
parser.add_argument(
"--header-file",
type=argparse.FileType("r"),
help=
"File from which to read a header, which will be printed before the "
"generated config.",
)
args = parser.parse_args()
report_stats = args.report_stats
if report_stats is not None:
report_stats = report_stats == "yes"
conf = HomeServerConfig().generate_config(
config_dir_path=args.config_dir,
data_dir_path=args.data_dir,
server_name=args.server_name,
generate_secrets=args.generate_secrets,
report_stats=report_stats,
)
if args.header_file:
shutil.copyfileobj(args.header_file, args.output_file)
args.output_file.write(conf)
def main() -> None:
parser = argparse.ArgumentParser(
description="A script to port an existing synapse SQLite database to"
" a new PostgreSQL database.")
parser.add_argument("-v", action="store_true")
parser.add_argument(
"--sqlite-database",
required=True,
help="The snapshot of the SQLite database file. This must not be"
" currently used by a running synapse server",
)
parser.add_argument(
"--postgres-config",
type=argparse.FileType("r"),
required=True,
help="The database config file for the PostgreSQL database",
)
parser.add_argument("--curses",
action="store_true",
help="display a curses based progress UI")
parser.add_argument(
"--batch-size",
type=int,
default=1000,
help="The number of rows to select from the SQLite table each"
" iteration [default=1000]",
)
args = parser.parse_args()
logging.basicConfig(
level=logging.DEBUG if args.v else logging.INFO,
format=
"%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(message)s",
filename="port-synapse.log" if args.curses else None,
)
sqlite_config = {
"name": "sqlite3",
"args": {
"database": args.sqlite_database,
"cp_min": 1,
"cp_max": 1,
"check_same_thread": False,
},
}
hs_config = yaml.safe_load(args.postgres_config)
if "database" not in hs_config:
sys.stderr.write(
"The configuration file must have a 'database' section.\n")
sys.exit(4)
postgres_config = hs_config["database"]
if "name" not in postgres_config:
sys.stderr.write("Malformed database config: no 'name'\n")
sys.exit(2)
if postgres_config["name"] != "psycopg2":
sys.stderr.write("Database must use the 'psycopg2' connector.\n")
sys.exit(3)
config = HomeServerConfig()
config.parse_config_dict(hs_config, "", "")
def start(stdscr: Optional["curses.window"] = None) -> None:
progress: Progress
if stdscr:
progress = CursesProgress(stdscr)
else:
progress = TerminalProgress()
porter = Porter(
sqlite_config=sqlite_config,
progress=progress,
batch_size=args.batch_size,
hs_config=config,
)
@defer.inlineCallbacks
def run() -> Generator["defer.Deferred[Any]", Any, None]:
with LoggingContext("synapse_port_db_run"):
yield defer.ensureDeferred(porter.run())
reactor.callWhenRunning(run)
reactor.run()
if args.curses:
curses.wrapper(start)
else:
start()
if end_error:
if end_error_exec_info:
exc_type, exc_value, exc_traceback = end_error_exec_info
traceback.print_exception(exc_type, exc_value, exc_traceback)
sys.stderr.write(end_error)
sys.exit(5)
