def __init__(self, reactor, cluster_state, configuration_service, endpoint,
context_factory):
"""
:param reactor: See ``ControlServiceLocator.__init__``.
:param ClusterStateService cluster_state: Object that records known
cluster state.
:param ConfigurationPersistenceService configuration_service:
Persistence service for desired cluster configuration.
:param endpoint: Endpoint to listen on.
:param context_factory: TLS context factory.
"""
self.connections = set()
self._reactor = reactor
self._connections_pending_update = set()
self._current_pending_update_delayed_call = None
self._current_command = {}
self.cluster_state = cluster_state
self.configuration_service = configuration_service
self.endpoint_service = StreamServerEndpointService(
endpoint,
TLSMemoryBIOFactory(
context_factory,
False,
ServerFactory.forProtocol(lambda: ControlAMP(reactor, self))
)
)
# When configuration changes, notify all connected clients:
self.configuration_service.register(self._schedule_broadcast_update)
def __init__(self, reactor, cluster_state, configuration_service, endpoint,
context_factory):
"""
:param reactor: See ``ControlServiceLocator.__init__``.
:param ClusterStateService cluster_state: Object that records known
cluster state.
:param ConfigurationPersistenceService configuration_service:
Persistence service for desired cluster configuration.
:param endpoint: Endpoint to listen on.
:param context_factory: TLS context factory.
"""
self.connections = set()
self._current_command = {}
self.cluster_state = cluster_state
self.configuration_service = configuration_service
self.endpoint_service = StreamServerEndpointService(
endpoint,
TLSMemoryBIOFactory(
context_factory,
False,
ServerFactory.forProtocol(lambda: ControlAMP(reactor, self))
)
)
# When configuration changes, notify all connected clients:
self.configuration_service.register(
lambda: self._send_state_to_connections(self.connections))
def _handshake(self, credential):
"""
Run a TLS handshake between a client and server, one of which is
using the validation logic and the other the given credential.
:param credential: The high-level credential to use.
:return ``Deferred``: Fires when handshake succeeded or
failed.
"""
peer_context_factory = PeerContextFactory(credential.credential)
port = find_free_port()[1]
validating_context_factory = context_factory_fixture(
port, self.good_ca)
if validator_is_client:
client_context_factory = validating_context_factory
server_context_factory = peer_context_factory
else:
server_context_factory = validating_context_factory
client_context_factory = peer_context_factory
server_endpoint = SSL4ServerEndpoint(reactor, port,
server_context_factory,
interface='127.0.0.1')
d = server_endpoint.listen(
ServerFactory.forProtocol(SendingProtocol))
d.addCallback(lambda port: self.addCleanup(port.stopListening))
validating_endpoint = SSL4ClientEndpoint(
reactor, "127.0.0.1", port, client_context_factory)
client_protocol = ReceivingProtocol()
result = connectProtocol(validating_endpoint, client_protocol)
result.addCallback(lambda _: client_protocol.result)
return result
def test_disconnectWhileProducing(self):
"""
If C{loseConnection} is called while a producer is registered with the
transport, the connection is closed after the producer is unregistered.
"""
reactor = self.buildReactor()
# For some reason, pyobject/pygtk will not deliver the close
# notification that should happen after the unregisterProducer call in
# this test. The selectable is in the write notification set, but no
# notification ever arrives. Probably for the same reason #5233 led
# win32eventreactor to be broken.
skippedReactors = ["Glib2Reactor", "Gtk2Reactor"]
reactorClassName = reactor.__class__.__name__
if reactorClassName in skippedReactors and platform.isWindows():
raise SkipTest("A pygobject/pygtk bug disables this functionality " "on Windows.")
class Producer:
def resumeProducing(self):
log.msg("Producer.resumeProducing")
self.listen(reactor, ServerFactory.forProtocol(Protocol))
finished = Deferred()
finished.addErrback(log.err)
finished.addCallback(lambda ign: reactor.stop())
class ClientProtocol(Protocol):
"""
Protocol to connect, register a producer, try to lose the
connection, unregister the producer, and wait for the connection to
actually be lost.
"""
def connectionMade(self):
log.msg("ClientProtocol.connectionMade")
self.transport.registerProducer(Producer(), False)
self.transport.loseConnection()
# Let the reactor tick over, in case synchronously calling
# loseConnection and then unregisterProducer is the same as
# synchronously calling unregisterProducer and then
# loseConnection (as it is in several reactors).
reactor.callLater(0, reactor.callLater, 0, self.unregister)
def unregister(self):
log.msg("ClientProtocol unregister")
self.transport.unregisterProducer()
# This should all be pretty quick. Fail the test
# if we don't get a connectionLost event really
# soon.
reactor.callLater(1.0, finished.errback, Failure(Exception("Connection was not lost")))
def connectionLost(self, reason):
log.msg("ClientProtocol.connectionLost")
finished.callback(None)
clientFactory = ClientFactory()
clientFactory.protocol = ClientProtocol
self.connect(reactor, clientFactory)
self.runReactor(reactor)
def __init__(self, reactor, cluster_state, configuration_service, endpoint,
context_factory):
"""
:param reactor: See ``ControlServiceLocator.__init__``.
:param ClusterStateService cluster_state: Object that records known
cluster state.
:param ConfigurationPersistenceService configuration_service:
Persistence service for desired cluster configuration.
:param endpoint: Endpoint to listen on.
:param context_factory: TLS context factory.
"""
self._connections = set()
self._reactor = reactor
self._connections_pending_update = set()
self._current_pending_update_delayed_call = None
self._current_command = {}
self._last_received_generation = defaultdict(
lambda: _ConfigAndStateGeneration())
self._configuration_generation_tracker = GenerationTracker(100)
self._state_generation_tracker = GenerationTracker(100)
self.cluster_state = cluster_state
self.configuration_service = configuration_service
self.endpoint_service = StreamServerEndpointService(
endpoint,
TLSMemoryBIOFactory(
context_factory, False,
ServerFactory.forProtocol(lambda: ControlAMP(reactor, self))))
# When configuration changes, notify all connected clients:
self.configuration_service.register(self._schedule_broadcast_update)
def test_connectEvent(self):
"""
This test checks that we correctly get notifications event for a
client. This ought to prevent a regression under Windows using the
GTK2 reactor. See #3925.
"""
reactor = self.buildReactor()
self.listen(reactor, ServerFactory.forProtocol(Protocol))
connected = []
class CheckConnection(Protocol):
def connectionMade(self):
connected.append(self)
reactor.stop()
clientFactory = Stop(reactor)
clientFactory.protocol = CheckConnection
needsRunningReactor(
reactor, lambda: self.connect(reactor, clientFactory))
reactor.run()
self.assertTrue(connected)
def test_connectEvent(self):
"""
This test checks that we correctly get notifications event for a
client. This ought to prevent a regression under Windows using the
GTK2 reactor. See #3925.
"""
reactor = self.buildReactor()
self.listen(reactor, ServerFactory.forProtocol(Protocol))
connected = []
class CheckConnection(Protocol):
def connectionMade(self):
connected.append(self)
reactor.stop()
clientFactory = Stop(reactor)
clientFactory.protocol = CheckConnection
needsRunningReactor(reactor,
lambda: self.connect(reactor, clientFactory))
reactor.run()
self.assertTrue(connected)
def __main__(argv=sys.argv):
from twisted.python.log import startLogging
startLogging(sys.stderr)
configuration: Configuration = cli.main(argv, standalone_mode=False).obj
reactor.listenUNIX(
configuration.listenPort,
ServerFactory.forProtocol(lambda: FDReceiverProtocol(configuration)))
return reactor.run()
def test_writeAfterDisconnect(self):
"""
After a connection is disconnected, L{ITransport.write} and
L{ITransport.writeSequence} are no-ops.
"""
reactor = self.buildReactor()
finished = []
serverConnectionLostDeferred = Deferred()
protocol = lambda: ClosingLaterProtocol(serverConnectionLostDeferred)
portDeferred = self.endpoints.server(reactor).listen(
ServerFactory.forProtocol(protocol))
def listening(port):
msg(f"Listening on {port.getHost()!r}")
endpoint = self.endpoints.client(reactor, port.getHost())
lostConnectionDeferred = Deferred()
protocol = lambda: ClosingLaterProtocol(lostConnectionDeferred)
client = endpoint.connect(ClientFactory.forProtocol(protocol))
def write(proto):
msg(f"About to write to {proto!r}")
proto.transport.write(b"x")
client.addCallbacks(write, lostConnectionDeferred.errback)
def disconnected(proto):
msg(f"{proto!r} disconnected")
proto.transport.write(b"some bytes to get lost")
proto.transport.writeSequence([b"some", b"more"])
finished.append(True)
lostConnectionDeferred.addCallback(disconnected)
serverConnectionLostDeferred.addCallback(disconnected)
return gatherResults(
[lostConnectionDeferred, serverConnectionLostDeferred])
def onListen():
portDeferred.addCallback(listening)
portDeferred.addErrback(err)
portDeferred.addCallback(lambda ignored: reactor.stop())
needsRunningReactor(reactor, onListen)
self.runReactor(reactor)
self.assertEqual(finished, [True, True])
def __init__(self, cluster_state, configuration_service, endpoint):
"""
:param ClusterStateService cluster_state: Object that records known
cluster state.
:param ConfigurationPersistenceService configuration_service:
Persistence service for desired cluster configuration.
:param endpoint: Endpoint to listen on.
"""
self.connections = set()
self.cluster_state = cluster_state
self.configuration_service = configuration_service
self.endpoint_service = StreamServerEndpointService(
endpoint, ServerFactory.forProtocol(lambda: ControlAMP(self)))
# When configuration changes, notify all connected clients:
self.configuration_service.register(
lambda: self._send_state_to_connections(self.connections))
def __init__(self, cluster_state, configuration_service, endpoint):
"""
:param ClusterStateService cluster_state: Object that records known
cluster state.
:param ConfigurationPersistenceService configuration_service:
Persistence service for desired cluster configuration.
:param endpoint: Endpoint to listen on.
"""
self.connections = set()
self.cluster_state = cluster_state
self.configuration_service = configuration_service
self.endpoint_service = StreamServerEndpointService(
endpoint, ServerFactory.forProtocol(lambda: ControlAMP(self)))
# When configuration changes, notify all connected clients:
self.configuration_service.register(
lambda: self._send_state_to_connections(self.connections))
def test_writeAfterDisconnect(self):
"""
After a connection is disconnected, L{ITransport.write} and
L{ITransport.writeSequence} are no-ops.
"""
reactor = self.buildReactor()
finished = []
serverConnectionLostDeferred = Deferred()
protocol = lambda: ClosingLaterProtocol(serverConnectionLostDeferred)
portDeferred = self.endpoints.server(reactor).listen(ServerFactory.forProtocol(protocol))
def listening(port):
msg("Listening on %r" % (port.getHost(),))
endpoint = self.endpoints.client(reactor, port.getHost())
lostConnectionDeferred = Deferred()
protocol = lambda: ClosingLaterProtocol(lostConnectionDeferred)
client = endpoint.connect(ClientFactory.forProtocol(protocol))
def write(proto):
msg("About to write to %r" % (proto,))
proto.transport.write(b"x")
client.addCallbacks(write, lostConnectionDeferred.errback)
def disconnected(proto):
msg("%r disconnected" % (proto,))
proto.transport.write(b"some bytes to get lost")
proto.transport.writeSequence([b"some", b"more"])
finished.append(True)
lostConnectionDeferred.addCallback(disconnected)
serverConnectionLostDeferred.addCallback(disconnected)
return gatherResults([lostConnectionDeferred, serverConnectionLostDeferred])
def onListen():
portDeferred.addCallback(listening)
portDeferred.addErrback(err)
portDeferred.addCallback(lambda ignored: reactor.stop())
needsRunningReactor(reactor, onListen)
self.runReactor(reactor)
self.assertEqual(finished, [True, True])
def test_unregisterProducerAfterDisconnect(self):
"""
If a producer is unregistered from a transport after the transport has
been disconnected (by the peer) and after C{loseConnection} has been
called, the transport is not re-added to the reactor as a writer as
would be necessary if the transport were still connected.
"""
reactor = self.buildReactor()
self.listen(reactor, ServerFactory.forProtocol(ClosingProtocol))
finished = Deferred()
finished.addErrback(log.err)
finished.addCallback(lambda ign: reactor.stop())
writing = []
class ClientProtocol(Protocol):
"""
Protocol to connect, register a producer, try to lose the
connection, wait for the server to disconnect from us, and then
unregister the producer.
"""
def connectionMade(self):
log.msg("ClientProtocol.connectionMade")
self.transport.registerProducer(
_SimplePullProducer(self.transport), False)
self.transport.loseConnection()
def connectionLost(self, reason):
log.msg("ClientProtocol.connectionLost")
self.unregister()
writing.append(self.transport in _getWriters(reactor))
finished.callback(None)
def unregister(self):
log.msg("ClientProtocol unregister")
self.transport.unregisterProducer()
clientFactory = ClientFactory()
clientFactory.protocol = ClientProtocol
self.connect(reactor, clientFactory)
self.runReactor(reactor)
self.assertFalse(writing[0],
"Transport was writing after unregisterProducer.")
def test_protocolGarbageAfterLostConnection(self):
"""
After the connection a protocol is being used for is closed, the
reactor discards all of its references to the protocol.
"""
lostConnectionDeferred = Deferred()
clientProtocol = ClosingLaterProtocol(lostConnectionDeferred)
clientRef = ref(clientProtocol)
reactor = self.buildReactor()
portDeferred = self.endpoints.server(reactor).listen(
ServerFactory.forProtocol(Protocol))
def listening(port):
msg(f"Listening on {port.getHost()!r}")
endpoint = self.endpoints.client(reactor, port.getHost())
client = endpoint.connect(
ClientFactory.forProtocol(lambda: clientProtocol))
def disconnect(proto):
msg(f"About to disconnect {proto!r}")
proto.transport.loseConnection()
client.addCallback(disconnect)
client.addErrback(lostConnectionDeferred.errback)
return lostConnectionDeferred
def onListening():
portDeferred.addCallback(listening)
portDeferred.addErrback(err)
portDeferred.addBoth(lambda ignored: reactor.stop())
needsRunningReactor(reactor, onListening)
self.runReactor(reactor)
# Drop the reference and get the garbage collector to tell us if there
# are no references to the protocol instance left in the reactor.
clientProtocol = None
collect()
self.assertIsNone(clientRef())
def test_protocolGarbageAfterLostConnection(self):
"""
After the connection a protocol is being used for is closed, the
reactor discards all of its references to the protocol.
"""
lostConnectionDeferred = Deferred()
clientProtocol = ClosingLaterProtocol(lostConnectionDeferred)
clientRef = ref(clientProtocol)
reactor = self.buildReactor()
portDeferred = self.endpoints.server(reactor).listen(ServerFactory.forProtocol(Protocol))
def listening(port):
msg("Listening on %r" % (port.getHost(),))
endpoint = self.endpoints.client(reactor, port.getHost())
client = endpoint.connect(ClientFactory.forProtocol(lambda: clientProtocol))
def disconnect(proto):
msg("About to disconnect %r" % (proto,))
proto.transport.loseConnection()
client.addCallback(disconnect)
client.addErrback(lostConnectionDeferred.errback)
return lostConnectionDeferred
def onListening():
portDeferred.addCallback(listening)
portDeferred.addErrback(err)
portDeferred.addBoth(lambda ignored: reactor.stop())
needsRunningReactor(reactor, onListening)
self.runReactor(reactor)
# Drop the reference and get the garbage collector to tell us if there
# are no references to the protocol instance left in the reactor.
clientProtocol = None
collect()
self.assertIdentical(None, clientRef())
def can_listen_tcp(port, interface=''):
"""Attempts to create a tcp listener on a port/interface combo
Args:
port (int): Listening port
interface (str): Hostname to bind to. If not specified binds to all
Returns:
ListenResult: the result of the test
"""
connection_server_factory = ServerFactory.forProtocol(Protocol)
try:
listener = reactor.listenTCP(
port=port,
factory=connection_server_factory,
interface=interface,
)
yield listener.stopListening()
result = ListenResult(True, port, interface)
except CannotListenError as e:
result = _reason_port_is_used(port, interface, e, 'tcp')
defer.returnValue(result)
def can_listen_ssl(port, ssl_ctx_factory, interface=''):
"""Attempts to create an ssl listener on a port/interface combo
Args:
port (int): Listening port
ssl_ctx_factory (twisted.internet.ssl.ContextFactory): Factory that can create the ssl contexts to be used by the connections
interface (str): Hostname to bind to. If not specified binds to all
Returns:
ListenResult: the result of the test
"""
connection_server_factory = ServerFactory.forProtocol(Protocol)
try:
listener = reactor.listenSSL(
port=port,
factory=connection_server_factory,
interface=interface,
contextFactory=ssl_ctx_factory,
)
yield listener.stopListening()
result = ListenResult(True, port, interface)
except CannotListenError as e:
result = _reason_port_is_used(port, interface, e, 'ssl')
defer.returnValue(result)
def test_disconnectWhileProducing(self):
"""
If C{loseConnection} is called while a producer is registered with the
transport, the connection is closed after the producer is unregistered.
"""
reactor = self.buildReactor()
# For some reason, pyobject/pygtk will not deliver the close
# notification that should happen after the unregisterProducer call in
# this test. The selectable is in the write notification set, but no
# notification ever arrives. Probably for the same reason #5233 led
# win32eventreactor to be broken.
skippedReactors = ["Glib2Reactor", "Gtk2Reactor"]
reactorClassName = reactor.__class__.__name__
if reactorClassName in skippedReactors and platform.isWindows():
raise SkipTest("A pygobject/pygtk bug disables this functionality "
"on Windows.")
class Producer:
def resumeProducing(self):
log.msg("Producer.resumeProducing")
self.listen(reactor, ServerFactory.forProtocol(Protocol))
finished = Deferred()
finished.addErrback(log.err)
finished.addCallback(lambda ign: reactor.stop())
class ClientProtocol(Protocol):
"""
Protocol to connect, register a producer, try to lose the
connection, unregister the producer, and wait for the connection to
actually be lost.
"""
def connectionMade(self):
log.msg("ClientProtocol.connectionMade")
self.transport.registerProducer(Producer(), False)
self.transport.loseConnection()
# Let the reactor tick over, in case synchronously calling
# loseConnection and then unregisterProducer is the same as
# synchronously calling unregisterProducer and then
# loseConnection (as it is in several reactors).
reactor.callLater(0, reactor.callLater, 0, self.unregister)
def unregister(self):
log.msg("ClientProtocol unregister")
self.transport.unregisterProducer()
# This should all be pretty quick. Fail the test
# if we don't get a connectionLost event really
# soon.
reactor.callLater(
1.0, finished.errback,
Failure(Exception("Connection was not lost")))
def connectionLost(self, reason):
log.msg("ClientProtocol.connectionLost")
finished.callback(None)
clientFactory = ClientFactory()
clientFactory.protocol = ClientProtocol
self.connect(reactor, clientFactory)
self.runReactor(reactor)
def setUp(self):
self.client_factory = ClientFactory.forProtocol(DummyClientProtocol)
self.server_factory = ServerFactory.forProtocol(DummyServerProtocol)
def for_protocol(cls, protocol, *args, **kw):
factory = ServerFactory.forProtocol(protocol)
return cls(factory, *args, **kw)
async def main(reactor, loops):
"""
Benchmark how long it takes to send `loops` messages.
"""
servers = []
def protocol():
p = LineCounter()
servers.append(p)
return p
logger_factory = ServerFactory.forProtocol(protocol)
logger_factory.wait_for = loops
logger_factory.on_done = Deferred()
port = reactor.listenTCP(0, logger_factory, interface="127.0.0.1")
# A fake homeserver config.
class Config:
server_name = "synmark-" + str(loops)
no_redirect_stdio = True
hs_config = Config()
# To be able to sleep.
clock = Clock(reactor)
errors = StringIO()
publisher = LogPublisher()
mock_sys = Mock()
beginner = LogBeginner(publisher,
errors,
mock_sys,
warnings,
initialBufferSize=loops)
log_config = {
"version": 1,
"loggers": {
"synapse": {
"level": "DEBUG",
"handlers": ["tersejson"]
}
},
"formatters": {
"tersejson": {
"class": "synapse.logging.TerseJsonFormatter"
}
},
"handlers": {
"tersejson": {
"class": "synapse.logging.RemoteHandler",
"host": "127.0.0.1",
"port": port.getHost().port,
"maximum_buffer": 100,
"_reactor": reactor,
}
},
}
logger = logging.getLogger("synapse.logging.test_terse_json")
_setup_stdlib_logging(
hs_config,
log_config,
logBeginner=beginner,
)
# Wait for it to connect...
for handler in logging.getLogger("synapse").handlers:
if isinstance(handler, RemoteHandler):
break
else:
raise RuntimeError("Improperly configured: no RemoteHandler found.")
await handler._service.whenConnected()
start = perf_counter()
# Send a bunch of useful messages
for i in range(0, loops):
logger.info("test message %s", i)
if len(handler._buffer) == handler.maximum_buffer:
while len(handler._buffer) > handler.maximum_buffer / 2:
await clock.sleep(0.01)
await logger_factory.on_done
end = perf_counter() - start
handler.close()
port.stopListening()
return end
def setUp(self):
self.client_factory = ClientFactory.forProtocol(DummyClientProtocol)
self.server_factory = ServerFactory.forProtocol(DummyServerProtocol)
async def main(reactor, loops):
"""
Benchmark how long it takes to send `loops` messages.
"""
servers = []
def protocol():
p = LineCounter()
servers.append(p)
return p
logger_factory = ServerFactory.forProtocol(protocol)
logger_factory.wait_for = loops
logger_factory.on_done = Deferred()
port = reactor.listenTCP(0, logger_factory, interface="127.0.0.1")
hs, wait, cleanup = await make_homeserver(reactor)
errors = StringIO()
publisher = LogPublisher()
mock_sys = Mock()
beginner = LogBeginner(publisher,
errors,
mock_sys,
warnings,
initialBufferSize=loops)
log_config = {
"loggers": {
"synapse": {
"level": "DEBUG"
}
},
"drains": {
"tersejson": {
"type": "network_json_terse",
"host": "127.0.0.1",
"port": port.getHost().port,
"maximum_buffer": 100,
}
},
}
logger = Logger(namespace="synapse.logging.test_terse_json",
observer=publisher)
logging_system = setup_structured_logging(hs,
hs.config,
log_config,
logBeginner=beginner,
redirect_stdlib_logging=False)
# Wait for it to connect...
await logging_system._observers[0]._service.whenConnected()
start = perf_counter()
# Send a bunch of useful messages
for i in range(0, loops):
logger.info("test message %s" % (i, ))
if (len(logging_system._observers[0]._buffer) ==
logging_system._observers[0].maximum_buffer):
while (len(logging_system._observers[0]._buffer) >
logging_system._observers[0].maximum_buffer / 2):
await wait(0.01)
await logger_factory.on_done
end = perf_counter() - start
logging_system.stop()
port.stopListening()
cleanup()
return end
def makeService(self, options):
factory = ServerFactory.forProtocol(options["protocol"])
service = ChildService(options["inherited-fd"], factory)
return service
def for_protocol(cls, protocol, *args, **kw):
factory = ServerFactory.forProtocol(protocol)
return cls(factory, *args, **kw)
factory.protocol = UpstreamProtocol
factory.server = self
reactor.connectTCP(SERVER_ADDRESS, SERVER_PORT, factory)
def dataReceived(self, data):
if self.client:
self.client.write(data)
else:
self._buffer.append(data)
def write(self, data):
self.transport.write(data)
class UpstreamProtocol(Protocol):
def connectionMade(self):
self.factory.server.client = self
for b in self.factory.server._buffer:
self.write(b)
self.factory.server._buffer.clear()
def dataReceived(self, data):
self.factory.server.write(data)
def write(self, data):
if data:
self.transport.write(data)
application = ServerFactory.forProtocol(DownstreamProtocol)
