def make_worker_hs(self, worker_app: str, extra_config: dict = {}, **kwargs) -> HomeServer: """Make a new worker HS instance, correctly connecting replcation stream to the master HS. Args: worker_app: Type of worker, e.g. `synapse.app.federation_sender`. extra_config: Any extra config to use for this instances. **kwargs: Options that get passed to `self.setup_test_homeserver`, useful to e.g. pass some mocks for things like `http_client` Returns: The new worker HomeServer instance. """ config = self._get_worker_hs_config() config["worker_app"] = worker_app config.update(extra_config) worker_hs = self.setup_test_homeserver( homeserverToUse=GenericWorkerServer, config=config, reactor=self.reactor, **kwargs) store = worker_hs.get_datastore() store.db_pool._db_pool = self.database_pool._db_pool repl_handler = ReplicationCommandHandler(worker_hs) client = ClientReplicationStreamProtocol( worker_hs, "client", "test", self.clock, repl_handler, ) server = self.server_factory.buildProtocol(None) client_transport = FakeTransport(server, self.reactor) client.makeConnection(client_transport) server_transport = FakeTransport(client, self.reactor) server.makeConnection(server_transport) # Set up a resource for the worker resource = ReplicationRestResource(self.hs) for servlet in self.servlets: servlet(worker_hs, resource) self._worker_hs_to_resource[worker_hs] = resource return worker_hs
def create_test_resource(self): """Overrides `HomeserverTestCase.create_test_resource`.""" # We override this so that it automatically registers all the HTTP # replication servlets, without having to explicitly do that in all # subclassses. resource = ReplicationRestResource(self.hs) for servlet in self.servlets: servlet(self.hs, resource) return resource
def _listen_http(self, listener_config: ListenerConfig): port = listener_config.port bind_addresses = listener_config.bind_addresses assert listener_config.http_options is not None site_tag = listener_config.http_options.tag if site_tag is None: site_tag = port # We always include a health resource. resources = {"/health": HealthResource()} for res in listener_config.http_options.resources: for name in res.names: if name == "metrics": resources[METRICS_PREFIX] = MetricsResource(RegistryProxy) elif name == "client": resource = JsonResource(self, canonical_json=False) RegisterRestServlet(self).register(resource) LoginRestServlet(self).register(resource) ThreepidRestServlet(self).register(resource) DevicesRestServlet(self).register(resource) KeyQueryServlet(self).register(resource) OneTimeKeyServlet(self).register(resource) KeyChangesServlet(self).register(resource) VoipRestServlet(self).register(resource) PushRuleRestServlet(self).register(resource) VersionsRestServlet(self).register(resource) ProfileAvatarURLRestServlet(self).register(resource) ProfileDisplaynameRestServlet(self).register(resource) ProfileRestServlet(self).register(resource) KeyUploadServlet(self).register(resource) AccountDataServlet(self).register(resource) RoomAccountDataServlet(self).register(resource) sync.register_servlets(self, resource) events.register_servlets(self, resource) room.register_servlets(self, resource, True) room.register_deprecated_servlets(self, resource) InitialSyncRestServlet(self).register(resource) room_keys.register_servlets(self, resource) tags.register_servlets(self, resource) account_data.register_servlets(self, resource) receipts.register_servlets(self, resource) read_marker.register_servlets(self, resource) SendToDeviceRestServlet(self).register(resource) user_directory.register_servlets(self, resource) # If presence is disabled, use the stub servlet that does # not allow sending presence if not self.config.use_presence: PresenceStatusStubServlet(self).register(resource) groups.register_servlets(self, resource) resources.update({CLIENT_API_PREFIX: resource}) elif name == "federation": resources.update( {FEDERATION_PREFIX: TransportLayerServer(self)}) elif name == "media": if self.config.can_load_media_repo: media_repo = self.get_media_repository_resource() # We need to serve the admin servlets for media on the # worker. admin_resource = JsonResource(self, canonical_json=False) register_servlets_for_media_repo(self, admin_resource) resources.update({ MEDIA_PREFIX: media_repo, LEGACY_MEDIA_PREFIX: media_repo, "/_synapse/admin": admin_resource, }) else: logger.warning( "A 'media' listener is configured but the media" " repository is disabled. Ignoring.") if name == "openid" and "federation" not in res.names: # Only load the openid resource separately if federation resource # is not specified since federation resource includes openid # resource. resources.update({ FEDERATION_PREFIX: TransportLayerServer(self, servlet_groups=["openid"]) }) if name in ["keys", "federation"]: resources[SERVER_KEY_V2_PREFIX] = KeyApiV2Resource(self) if name == "replication": resources[REPLICATION_PREFIX] = ReplicationRestResource( self) root_resource = create_resource_tree(resources, OptionsResource()) _base.listen_tcp( bind_addresses, port, SynapseSite( "synapse.access.http.%s" % (site_tag, ), site_tag, listener_config, root_resource, self.version_string, ), reactor=self.get_reactor(), ) logger.info("Synapse worker now listening on port %d", port)
def _configure_named_resource(self, name, compress=False): """Build a resource map for a named resource Args: name (str): named resource: one of "client", "federation", etc compress (bool): whether to enable gzip compression for this resource Returns: dict[str, Resource]: map from path to HTTP resource """ resources = {} if name == "client": client_resource = ClientRestResource(self) if compress: client_resource = gz_wrap(client_resource) resources.update({ "/_matrix/client/api/v1": client_resource, "/_matrix/client/r0": client_resource, "/_matrix/client/unstable": client_resource, "/_matrix/client/v2_alpha": client_resource, "/_matrix/client/versions": client_resource, "/.well-known/matrix/client": WellKnownResource(self), "/_synapse/admin": AdminRestResource(self), "/_synapse/client/pick_username": pick_username_resource(self), "/_synapse/client/pick_idp": PickIdpResource(self), }) if self.get_config().oidc_enabled: from synapse.rest.oidc import OIDCResource resources["/_synapse/oidc"] = OIDCResource(self) if self.get_config().saml2_enabled: from synapse.rest.saml2 import SAML2Resource resources["/_matrix/saml2"] = SAML2Resource(self) if self.get_config( ).threepid_behaviour_email == ThreepidBehaviour.LOCAL: from synapse.rest.synapse.client.password_reset import ( PasswordResetSubmitTokenResource, ) resources[ "/_synapse/client/password_reset/email/submit_token"] = PasswordResetSubmitTokenResource( self) if name == "consent": from synapse.rest.consent.consent_resource import ConsentResource consent_resource = ConsentResource(self) if compress: consent_resource = gz_wrap(consent_resource) resources.update({"/_matrix/consent": consent_resource}) if name == "federation": resources.update({FEDERATION_PREFIX: TransportLayerServer(self)}) if name == "openid": resources.update({ FEDERATION_PREFIX: TransportLayerServer(self, servlet_groups=["openid"]) }) if name in ["static", "client"]: resources.update({ STATIC_PREFIX: StaticResource( os.path.join(os.path.dirname(synapse.__file__), "static")) }) if name in ["media", "federation", "client"]: if self.get_config().enable_media_repo: media_repo = self.get_media_repository_resource() resources.update({ MEDIA_PREFIX: media_repo, LEGACY_MEDIA_PREFIX: media_repo }) elif name == "media": raise ConfigError( "'media' resource conflicts with enable_media_repo=False") if name in ["keys", "federation"]: resources[SERVER_KEY_V2_PREFIX] = KeyApiV2Resource(self) if name == "webclient": webclient_loc = self.get_config().web_client_location if webclient_loc is None: logger.warning( "Not enabling webclient resource, as web_client_location is unset." ) elif webclient_loc.startswith( "http://") or webclient_loc.startswith("https://"): resources[WEB_CLIENT_PREFIX] = RootRedirect(webclient_loc) else: logger.warning( "Running webclient on the same domain is not recommended: " "https://github.com/matrix-org/synapse#security-note - " "after you move webclient to different host you can set " "web_client_location to its full URL to enable redirection." ) # GZip is disabled here due to # https://twistedmatrix.com/trac/ticket/7678 resources[WEB_CLIENT_PREFIX] = File(webclient_loc) if name == "metrics" and self.get_config().enable_metrics: resources[METRICS_PREFIX] = MetricsResource(RegistryProxy) if name == "replication": resources[REPLICATION_PREFIX] = ReplicationRestResource(self) return resources
def make_worker_hs(self, worker_app: str, extra_config: dict = {}, **kwargs) -> HomeServer: """Make a new worker HS instance, correctly connecting replcation stream to the master HS. Args: worker_app: Type of worker, e.g. `synapse.app.federation_sender`. extra_config: Any extra config to use for this instances. **kwargs: Options that get passed to `self.setup_test_homeserver`, useful to e.g. pass some mocks for things like `http_client` Returns: The new worker HomeServer instance. """ config = self._get_worker_hs_config() config["worker_app"] = worker_app config.update(extra_config) worker_hs = self.setup_test_homeserver( homeserver_to_use=GenericWorkerServer, config=config, reactor=self.reactor, **kwargs, ) # If the instance is in the `instance_map` config then workers may try # and send HTTP requests to it, so we register it with # `_handle_http_replication_attempt` like we do with the master HS. instance_name = worker_hs.get_instance_name() instance_loc = worker_hs.config.worker.instance_map.get(instance_name) if instance_loc: # Ensure the host is one that has a fake DNS entry. if instance_loc.host not in self.reactor.lookups: raise Exception( "Host does not have an IP for instance_map[%r].host = %r" % ( instance_name, instance_loc.host, )) self.reactor.add_tcp_client_callback( self.reactor.lookups[instance_loc.host], instance_loc.port, lambda: self._handle_http_replication_attempt( worker_hs, instance_loc.port), ) store = worker_hs.get_datastore() store.db_pool._db_pool = self.database_pool._db_pool # Set up TCP replication between master and the new worker if we don't # have Redis support enabled. if not worker_hs.config.redis_enabled: repl_handler = ReplicationCommandHandler(worker_hs) client = ClientReplicationStreamProtocol( worker_hs, "client", "test", self.clock, repl_handler, ) server = self.server_factory.buildProtocol(None) client_transport = FakeTransport(server, self.reactor) client.makeConnection(client_transport) server_transport = FakeTransport(client, self.reactor) server.makeConnection(server_transport) # Set up a resource for the worker resource = ReplicationRestResource(worker_hs) for servlet in self.servlets: servlet(worker_hs, resource) self._hs_to_site[worker_hs] = SynapseSite( logger_name="synapse.access.http.fake", site_tag="{}-{}".format(worker_hs.config.server.server_name, worker_hs.get_instance_name()), config=worker_hs.config.server.listeners[0], resource=resource, server_version_string="1", ) if worker_hs.config.redis.redis_enabled: worker_hs.get_tcp_replication().start_replication(worker_hs) return worker_hs
def _configure_named_resource(self, name, compress=False): """Build a resource map for a named resource Args: name (str): named resource: one of "client", "federation", etc compress (bool): whether to enable gzip compression for this resource Returns: dict[str, Resource]: map from path to HTTP resource """ resources = {} if name == "client": client_resource = ClientRestResource(self) if compress: client_resource = gz_wrap(client_resource) resources.update({ "/_matrix/client/api/v1": client_resource, "/_matrix/client/r0": client_resource, "/_matrix/client/unstable": client_resource, "/_matrix/client/v2_alpha": client_resource, "/_matrix/client/versions": client_resource, "/.well-known/matrix/client": WellKnownResource(self), }) if self.get_config().saml2_enabled: from synapse.rest.saml2 import SAML2Resource resources["/_matrix/saml2"] = SAML2Resource(self) if name == "consent": from synapse.rest.consent.consent_resource import ConsentResource consent_resource = ConsentResource(self) if compress: consent_resource = gz_wrap(consent_resource) resources.update({ "/_matrix/consent": consent_resource, }) if name == "federation": resources.update({ FEDERATION_PREFIX: TransportLayerServer(self), }) if name == "openid": resources.update({ FEDERATION_PREFIX: TransportLayerServer(self, servlet_groups=["openid"]), }) if name in ["static", "client"]: resources.update({ STATIC_PREFIX: File(os.path.join(os.path.dirname(synapse.__file__), "static")), }) if name in ["media", "federation", "client"]: if self.get_config().enable_media_repo: media_repo = self.get_media_repository_resource() resources.update({ MEDIA_PREFIX: media_repo, LEGACY_MEDIA_PREFIX: media_repo, CONTENT_REPO_PREFIX: ContentRepoResource(self, self.config.uploads_path), }) elif name == "media": raise ConfigError( "'media' resource conflicts with enable_media_repo=False", ) if name in ["keys", "federation"]: resources[SERVER_KEY_V2_PREFIX] = KeyApiV2Resource(self) if name == "webclient": webclient_path = self.get_config().web_client_location if webclient_path is None: logger.warning( "Not enabling webclient resource, as web_client_location is unset." ) else: # GZip is disabled here due to # https://twistedmatrix.com/trac/ticket/7678 resources[WEB_CLIENT_PREFIX] = File(webclient_path) if name == "metrics" and self.get_config().enable_metrics: resources[METRICS_PREFIX] = MetricsResource(RegistryProxy) if name == "replication": resources[REPLICATION_PREFIX] = ReplicationRestResource(self) return resources
def _listen_http(self, listener_config: ListenerConfig) -> None: port = listener_config.port bind_addresses = listener_config.bind_addresses assert listener_config.http_options is not None site_tag = listener_config.http_options.tag if site_tag is None: site_tag = str(port) # We always include a health resource. resources: Dict[str, Resource] = {"/health": HealthResource()} for res in listener_config.http_options.resources: for name in res.names: if name == "metrics": resources[METRICS_PREFIX] = MetricsResource(RegistryProxy) elif name == "client": resource = JsonResource(self, canonical_json=False) RegisterRestServlet(self).register(resource) RegistrationTokenValidityRestServlet(self).register(resource) login.register_servlets(self, resource) ThreepidRestServlet(self).register(resource) WhoamiRestServlet(self).register(resource) DevicesRestServlet(self).register(resource) # Read-only KeyUploadServlet(self).register(resource) KeyQueryServlet(self).register(resource) KeyChangesServlet(self).register(resource) OneTimeKeyServlet(self).register(resource) voip.register_servlets(self, resource) push_rule.register_servlets(self, resource) versions.register_servlets(self, resource) profile.register_servlets(self, resource) sync.register_servlets(self, resource) events.register_servlets(self, resource) room.register_servlets(self, resource, is_worker=True) room.register_deprecated_servlets(self, resource) initial_sync.register_servlets(self, resource) room_batch.register_servlets(self, resource) room_keys.register_servlets(self, resource) tags.register_servlets(self, resource) account_data.register_servlets(self, resource) receipts.register_servlets(self, resource) read_marker.register_servlets(self, resource) sendtodevice.register_servlets(self, resource) user_directory.register_servlets(self, resource) presence.register_servlets(self, resource) resources.update({CLIENT_API_PREFIX: resource}) resources.update(build_synapse_client_resource_tree(self)) resources.update({"/.well-known": well_known_resource(self)}) elif name == "federation": resources.update({FEDERATION_PREFIX: TransportLayerServer(self)}) elif name == "media": if self.config.media.can_load_media_repo: media_repo = self.get_media_repository_resource() # We need to serve the admin servlets for media on the # worker. admin_resource = JsonResource(self, canonical_json=False) register_servlets_for_media_repo(self, admin_resource) resources.update( { MEDIA_R0_PREFIX: media_repo, MEDIA_V3_PREFIX: media_repo, LEGACY_MEDIA_PREFIX: media_repo, "/_synapse/admin": admin_resource, } ) else: logger.warning( "A 'media' listener is configured but the media" " repository is disabled. Ignoring." ) if name == "openid" and "federation" not in res.names: # Only load the openid resource separately if federation resource # is not specified since federation resource includes openid # resource. resources.update( { FEDERATION_PREFIX: TransportLayerServer( self, servlet_groups=["openid"] ) } ) if name in ["keys", "federation"]: resources[SERVER_KEY_V2_PREFIX] = KeyApiV2Resource(self) if name == "replication": resources[REPLICATION_PREFIX] = ReplicationRestResource(self) # Attach additional resources registered by modules. resources.update(self._module_web_resources) self._module_web_resources_consumed = True root_resource = create_resource_tree(resources, OptionsResource()) _base.listen_tcp( bind_addresses, port, SynapseSite( "synapse.access.http.%s" % (site_tag,), site_tag, listener_config, root_resource, self.version_string, max_request_body_size=max_request_body_size(self.config), reactor=self.get_reactor(), ), reactor=self.get_reactor(), ) logger.info("Synapse worker now listening on port %d", port)
def _configure_named_resource(self, name, compress=False): """Build a resource map for a named resource Args: name (str): named resource: one of "client", "federation", etc compress (bool): whether to enable gzip compression for this resource Returns: dict[str, Resource]: map from path to HTTP resource """ resources = {} if name == "client": client_resource = ClientRestResource(self) if compress: client_resource = gz_wrap(client_resource) resources.update({ "/_matrix/client/api/v1": client_resource, "/_matrix/client/r0": client_resource, "/_matrix/client/unstable": client_resource, "/_matrix/client/v2_alpha": client_resource, "/_matrix/client/versions": client_resource, }) if name == "federation": resources.update({ FEDERATION_PREFIX: TransportLayerServer(self), }) if name in ["static", "client"]: resources.update({ STATIC_PREFIX: File(os.path.join(os.path.dirname(synapse.__file__), "static")), }) if name in ["media", "federation", "client"]: if self.get_config().enable_media_repo: media_repo = self.get_media_repository_resource() resources.update({ MEDIA_PREFIX: media_repo, LEGACY_MEDIA_PREFIX: media_repo, CONTENT_REPO_PREFIX: ContentRepoResource(self, self.config.uploads_path), }) elif name == "media": raise ConfigError( "'media' resource conflicts with enable_media_repo=False", ) if name in ["keys", "federation"]: resources.update({ SERVER_KEY_PREFIX: LocalKey(self), SERVER_KEY_V2_PREFIX: KeyApiV2Resource(self), }) if name == "webclient": resources[WEB_CLIENT_PREFIX] = build_resource_for_web_client(self) if name == "metrics" and self.get_config().enable_metrics: resources[METRICS_PREFIX] = MetricsResource(self) if name == "replication": resources[REPLICATION_PREFIX] = ReplicationRestResource(self) return resources
def create_resource_dict(self) -> Dict[str, Resource]: d = super().create_resource_dict() d["/_synapse/replication"] = ReplicationRestResource(self.hs) return d
def _configure_named_resource(self, name: str, compress: bool = False ) -> Dict[str, Resource]: """Build a resource map for a named resource Args: name: named resource: one of "client", "federation", etc compress: whether to enable gzip compression for this resource Returns: map from path to HTTP resource """ resources: Dict[str, Resource] = {} if name == "client": client_resource: Resource = ClientRestResource(self) if compress: client_resource = gz_wrap(client_resource) resources.update({ CLIENT_API_PREFIX: client_resource, "/.well-known": well_known_resource(self), "/_synapse/admin": AdminRestResource(self), **build_synapse_client_resource_tree(self), }) if self.config.email.threepid_behaviour_email == ThreepidBehaviour.LOCAL: from synapse.rest.synapse.client.password_reset import ( PasswordResetSubmitTokenResource, ) resources[ "/_synapse/client/password_reset/email/submit_token"] = PasswordResetSubmitTokenResource( self) if name == "consent": from synapse.rest.consent.consent_resource import ConsentResource consent_resource: Resource = ConsentResource(self) if compress: consent_resource = gz_wrap(consent_resource) resources.update({"/_matrix/consent": consent_resource}) if name == "federation": resources.update({FEDERATION_PREFIX: TransportLayerServer(self)}) if name == "openid": resources.update({ FEDERATION_PREFIX: TransportLayerServer(self, servlet_groups=["openid"]) }) if name in ["static", "client"]: resources.update({ STATIC_PREFIX: StaticResource( os.path.join(os.path.dirname(synapse.__file__), "static")) }) if name in ["media", "federation", "client"]: if self.config.server.enable_media_repo: media_repo = self.get_media_repository_resource() resources.update({ MEDIA_R0_PREFIX: media_repo, MEDIA_V3_PREFIX: media_repo, LEGACY_MEDIA_PREFIX: media_repo, }) elif name == "media": raise ConfigError( "'media' resource conflicts with enable_media_repo=False") if name in ["keys", "federation"]: resources[SERVER_KEY_V2_PREFIX] = KeyApiV2Resource(self) if name == "metrics" and self.config.metrics.enable_metrics: metrics_resource: Resource = MetricsResource(RegistryProxy) if compress: metrics_resource = gz_wrap(metrics_resource) resources[METRICS_PREFIX] = metrics_resource if name == "replication": resources[REPLICATION_PREFIX] = ReplicationRestResource(self) return resources