def finish(r):
# By now the Agent should have all it needs to parse a response.
protocol.connectionLost(Failure(ConnectionDone()))
# Tell it that the connection is now complete so it can clean up.
channel.connectionLost(Failure(ConnectionDone()))
# Propogate the response.
return r
def request(self, method, uri, headers=None, bodyProducer=None):
"""
Implement IAgent.request.
"""
# We want to use Agent to parse the HTTP response, so let's ask it to
# make a request against our in-memory reactor.
response = self._realAgent.request(method, uri, headers, bodyProducer)
# That will try to establish an HTTP connection with the reactor's
# connectTCP method, and MemoryReactor will place Agent's factory into
# the tcpClients list. We'll extract that.
host, port, factory, timeout, bindAddress = (
self._memoryReactor.tcpClients[0])
# Then we need to convince that factory it's connected to something and
# it will give us a protocol for that connection.
protocol = factory.buildProtocol(None)
# We want to capture the output of that connection so we'll make an
# in-memory transport.
clientTransport = AbortableStringTransport()
# When the protocol is connected to a transport, it ought to send the
# whole request because callers of this should not use an asynchronous
# bodyProducer.
protocol.makeConnection(clientTransport)
# Get the data from the request.
requestData = clientTransport.io.getvalue()
# Now time for the server to do its job. Ask it to build an HTTP
# channel.
channel = Site(self._rootResource).buildProtocol(None)
# Connect the channel to another in-memory transport so we can collect
# the response.
serverTransport = StringTransport()
serverTransport.hostAddr = IPv4Address('TCP', '127.0.0.1', 80)
channel.makeConnection(serverTransport)
# Feed it the data that the Agent synthesized.
channel.dataReceived(requestData)
# Tell it that the connection is now complete so it can clean up.
channel.connectionLost(Failure(ConnectionDone()))
# Now we have the response data, let's give it back to the Agent.
protocol.dataReceived(serverTransport.io.getvalue())
# By now the Agent should have all it needs to parse a response.
protocol.connectionLost(Failure(ConnectionDone()))
# Return the response in the accepted format (Deferred firing
# IResponse). This should be synchronously fired, and if not, it's the
# system under test's problem.
return response
def test_connectionLostWithoutAvatar(self):
"""
L{CredReceiver.connectionLost} does not raise an exception if no login
has occurred when it is called.
"""
self.server.connectionLost(
Failure(ConnectionDone("test connection lost")))
def test_connectionClosedBeforeSecure(self):
"""
If the connection closes at any point before the SSH transport layer
has finished key exchange (ie, gotten to the point where we may attempt
to authenticate), the L{Deferred} returned by
L{SSHCommandClientEndpoint.connect} fires with a L{Failure} wrapping
the reason for the lost connection.
"""
endpoint = SSHCommandClientEndpoint.newConnection(
self.reactor,
b"/bin/ls -l",
b"dummy user",
self.hostname,
self.port,
knownHosts=self.knownHosts,
ui=FixedResponseUI(False))
factory = Factory()
factory.protocol = Protocol
d = endpoint.connect(factory)
transport = StringTransport()
factory = self.reactor.tcpClients[0][2]
client = factory.buildProtocol(None)
client.makeConnection(transport)
client.connectionLost(Failure(ConnectionDone()))
self.failureResultOf(d).trap(ConnectionDone)
def test_connectionCancelledBeforeSecure(self):
"""
If the connection is cancelled before the SSH transport layer has
finished key exchange (ie, gotten to the point where we may attempt to
authenticate), the L{Deferred} returned by
L{SSHCommandClientEndpoint.connect} fires with a L{Failure} wrapping
L{CancelledError} and the connection is aborted.
"""
endpoint = SSHCommandClientEndpoint.newConnection(
self.reactor,
b"/bin/ls -l",
b"dummy user",
self.hostname,
self.port,
knownHosts=self.knownHosts,
ui=FixedResponseUI(False))
factory = Factory()
factory.protocol = Protocol
d = endpoint.connect(factory)
transport = AbortableFakeTransport(None, isServer=False)
factory = self.reactor.tcpClients[0][2]
client = factory.buildProtocol(None)
client.makeConnection(transport)
d.cancel()
self.failureResultOf(d).trap(CancelledError)
self.assertTrue(transport.aborted)
# Make sure the connection closing doesn't result in unexpected
# behavior when due to cancellation:
client.connectionLost(Failure(ConnectionDone()))
def frombytes(cls, byte_string):
"""
Parse an AWS request from a byte string.
@param byte_string: The request as parsed from a C{.req} file.
@type byte_string: L{bytes}
@return: A request object.
@rtype: L{_AWSRequest}
"""
# Ensure there's a blank line so that status and header
# parsing completes.
blank_line = b'\n\n'
if blank_line not in byte_string:
byte_string += blank_line
channel = HTTPChannel()
channel.delimiter = b'\n'
channel.makeConnection(StringTransport())
channel.dataReceived(byte_string)
channel.connectionLost(ConnectionDone())
request = channel.requests[-1]
method = request.method
path = request.uri
headers = dict(request.requestHeaders.getAllRawHeaders())
# what comes after the empty line is a
body = byte_string.split(blank_line, 1)[-1]
return cls(method, path, headers, body)
def _finishResponse(self, rest):
"""
Called by an L{HTTPClientParser} to indicate that it has parsed a
complete response.
@param rest: A C{str} giving any trailing bytes which were given to
the L{HTTPClientParser} which were not part of the response it
was parsing.
"""
assert self._state in ('WAITING', 'TRANSMITTING')
if self._state == 'WAITING':
self._state = 'QUIESCENT'
else:
# The server sent the entire response before we could send the
# whole request. That sucks. Oh well. Fire the request()
# Deferred with the response. But first, make sure that if the
# request does ever finish being written that it won't try to fire
# that Deferred.
self._state = 'TRANSMITTING_AFTER_RECEIVING_RESPONSE'
self._responseDeferred.chainDeferred(self._finishedRequest)
reason = ConnectionDone("synthetic!")
connHeaders = self._parser.connHeaders.getRawHeaders('connection')
if (connHeaders is not None) and ('close' in connHeaders):
self._giveUp(Failure(reason))
else:
# It's persistent connection
self._disconnectParser(reason)
def test_kill_unexpected_exit(self):
"""
The :py:class:`Deferred` returned by
:py:meth:`_HTTPBinProcess.kill` errbacks with the failure when
it is not :py:class:`ProcessTerminated`, or its signal does
not match the expected signal.
"""
for error in [
ProcessTerminated(1, signal=signal.SIGIO),
ConnectionDone("Bye")
]:
httpbin_process = parent._HTTPBinProcess(https=False)
httpbin_process.server_description(self.reactor)
spawned_process = self.reactor.spawnedProcesses[-1]
spawned_process.send_stdout(
shared._HTTPBinDescription(host="host",
port=1234).to_json_bytes() + b'\n')
termination_deferred = httpbin_process.kill()
spawned_process.end_process(Failure(error))
self.assertIs(
self.failureResultOf(termination_deferred).value, error)
def _disconnected(self, reason):
"""
Called when the stream has been closed.
From this point on, the manager doesn't interact with the
L{XmlStream} anymore and notifies each handler that the connection
was lost by calling its C{connectionLost} method.
"""
self.xmlstream = None
self._initialized = False
# Twisted versions before 11.0 passed an XmlStream here.
if not hasattr(reason, 'trap'):
reason = failure.Failure(ConnectionDone())
# Notify all child services which implement
# the IService interface
for e in list(self):
e.connectionLost(reason)
# This errbacks all deferreds of iq's for which no response has
# been received with a L{ConnectionLost} failure. Otherwise, the
# deferreds will never be fired.
iqDeferreds = self._iqDeferreds
self._iqDeferreds = {}
for d in iqDeferreds.itervalues():
d.errback(reason)
def test_connection_done(test_client, mock_treq):
mock_treq.post.side_effect = ConnectionDone()
with pytest.raises(ConnectionDone):
yield test_client.request("POST",
"/test/uri",
SOME_JSON,
is_retry=True)
def test_disconnected(self):
"""
If the transport is disconnected before the response is available, no
``RuntimeError`` is logged for finishing a disconnected request.
"""
result = Deferred()
resource = StaticResource(result)
d = render(resource, {})
resource._request.connectionLost(Failure(ConnectionDone()))
result.callback(b"Some result")
self.assertThat(
d,
failed(
AfterPreprocessing(
lambda reason: reason.type,
Equals(ConnectionDone),
),
),
)
# Since we're not a trial TestCase we don't have flushLoggedErrors.
# The next best thing is to make sure any dangling Deferreds have been
# garbage collected and then let the generic trial logic for failing
# tests with logged errors kick in.
gc.collect()
def test_connectionMadeLost(self):
"""
C{connectionMade} and C{connectionLost} are called on the protocol by
the C{SerialPort}.
"""
# Serial port that doesn't actually connect to anything:
class DummySerialPort(serialport.SerialPort):
_serialFactory = DoNothing
def _finishPortSetup(self):
pass # override default win32 actions
events = []
class SerialProtocol(Protocol):
def connectionMade(self):
events.append("connectionMade")
def connectionLost(self, reason):
events.append(("connectionLost", reason))
# Creation of port should result in connectionMade call:
port = DummySerialPort(SerialProtocol(), "", reactor=DoNothing())
self.assertEqual(events, ["connectionMade"])
# Simulate reactor calling connectionLost on the SerialPort:
f = Failure(ConnectionDone())
port.connectionLost(f)
self.assertEqual(events, ["connectionMade", ("connectionLost", f)])
def _reader(self):
"""Wait forever for new chunks. Notify the user about the good ones through self._on_incoming_cb.
If a response to a previously sent chunk is received, pop the corresponding
response_cb from self.expected_responses and send the response there.
"""
error = Failure(ConnectionDone())
try:
self.writer_job.link(self.reader_job)
try:
while self.state in ['CONNECTED', 'FLUSHING']:
chunk = self.msrp.read_chunk()
if chunk.method is None: # response
self._handle_incoming_response(chunk)
else:
method = getattr(self,
'_handle_incoming_%s' % chunk.method,
None)
if method is not None:
method(chunk)
else:
response = make_response(chunk, 501,
'Method unknown')
self.outgoing.send(OutgoingChunk(response))
except proc.LinkedExited: # writer has exited
pass
finally:
self.writer_job.unlink(self.reader_job)
self.writer_job.kill()
self.logger.debug('reader: expecting responses only')
delay = time() - self.last_expected_response
if delay >= 0 and self.expected_responses:
# continue read the responses until the last timeout expires
with api.timeout(delay, None):
while self.expected_responses:
chunk = self.msrp.read_chunk()
if chunk.method is None:
self._handle_incoming_response(chunk)
else:
self.logger.debug('dropping incoming %r', chunk)
# read whatever left in the queue
with api.timeout(0, None):
while self.msrp._queue:
chunk = self.msrp.read_chunk()
if chunk.method is None:
self._handle_incoming_response(chunk)
else:
self.logger.debug('dropping incoming %r', chunk)
self.logger.debug('reader: done')
except ConnectionClosedErrors as e:
self.logger.debug('reader: exiting because of %r', e)
error = Failure(e)
except Exception:
self.logger.exception('reader: captured unhandled exception:')
error = Failure()
raise
finally:
self._on_incoming_cb(error=error)
self.msrp.loseConnection(wait=False)
self.set_state('DONE')
def test_stopTryingWhenConnected(self):
"""
If a L{KafkaBrokerClient} has C{stopTrying} called while it is
connected, it does not subsequently attempt to reconnect if the
connection is later lost.
"""
reactor = MemoryReactorClock()
class NoConnectConnector(object):
def stopConnecting(self):
raise ClientError("Shouldn't be called, "
"we're connected.") # pragma: no cover
def connect(self):
raise ClientError(
"Shouldn't be reconnecting.") # pragma: no cover
c = KafkaBrokerClient(reactor, 'broker', 9092, 'clientId')
c.protocol = Protocol
# Let's pretend we've connected:
c.buildProtocol(None)
# Now we stop trying, then disconnect:
c.stopTrying()
c.clientConnectionLost(NoConnectConnector(), Failure(ConnectionDone()))
self.assertFalse(c.continueTrying)
def close(self, reason=None, within=0):
"""Explicitely close the connection.
@param reason: Optional closing reason. If not given, ConnectionDone
will be used.
@param within: Shutdown the client within this amount of seconds. If
zero (the default), all channels and queues will be closed
immediately. If greater than 0, try to close the AMQP connection
cleanly, by sending a "close" method and waiting for "close-ok". If
no reply is received within the given amount of seconds, the
transport will be forcely shutdown.
"""
if self.closed:
return
if reason is None:
reason = ConnectionDone()
if within > 0:
channel0 = yield self.channel(0)
deferred = channel0.connection_close()
call = self.clock.callLater(within, deferred.cancel)
try:
yield deferred
except defer.CancelledError:
pass
else:
call.cancel()
self.do_close(reason)
def test_writeConnectionLost(self):
"""
The protocol created by L{flowFountFromEndpoint} provides half-close
support, and when it receives an EOF (i.e.: C{writeConnectionLost}) it
will signal the end of the flow to its drain's fount, but not to its
fount's drain.
"""
endpoint, ports = fakeEndpointWithPorts()
fffep = flowFountFromEndpoint(endpoint)
fffep.callback(None)
flowFount = self.successResultOf(fffep)
protocol = ports[0].factory.buildProtocol(None)
verifyObject(IHalfCloseableProtocol, protocol)
aTransport = StringTransport()
protocol.makeConnection(aTransport)
accepted = FakeDrain()
flowFount.flowTo(accepted)
[flow] = accepted.received
receivedData = FakeDrain()
dataSender = FakeFount()
flow.fount.flowTo(receivedData)
dataSender.flowTo(flow.drain)
self.assertEqual(len(receivedData.stopped), 0)
self.assertEqual(dataSender.flowIsStopped, False)
protocol.writeConnectionLost()
self.assertEqual(len(receivedData.stopped), 0)
self.assertEqual(dataSender.flowIsStopped, 1)
protocol.connectionLost(ConnectionDone())
self.assertEqual(len(receivedData.stopped), 1)
self.assertEqual(dataSender.flowIsStopped, 1)
def test_process_exception_outside_crawlera(self):
self.spider.crawlera_enabled = False
crawler = self._mock_crawler(self.spider, self.settings)
mw = self.mwcls.from_crawler(crawler)
mw.open_spider(self.spider)
req = Request("https://scrapy.org")
assert mw.process_exception(req, ConnectionDone(), self.spider) is None
def test_connection_err(self):
from twisted.internet.error import ConnectionDone
try:
raise ConnectionDone("Connection was closed cleanly.")
except Exception:
fail = Failure()
self.base._response_err(fail)
assert not self.status_mock.called
def test_connection_done(self):
# pretend we connected
self.proto._connectionMade()
self.proto.connectionLost(Failure(ConnectionDone()))
messages = ' '.join([str(x[1]) for x in self.proto.log.mock_calls])
self.assertTrue('closed cleanly' in messages)
def loseConnection(self):
if not self.connected:
return
self.connected = False
reason = Failure(ConnectionDone("Bye."))
self.protocol.connectionLost(reason)
if self.connector is not None:
self.connector.connectionLost(reason)
def test_requestDisconnectCleanup(self):
"""
Test if deferreds for iq's that haven't yet received a response
have their errback called on stream disconnect.
"""
d = self.streamManager.request(self.request)
xs = self.xmlstream
xs.connectionLost(failure.Failure(ConnectionDone()))
self.assertFailure(d, ConnectionDone)
return d
def test_connectionTracking(self):
"""
L{dns.DNSProtocol} calls its controller's C{connectionMade}
method with itself when it is connected to a transport and its
controller's C{connectionLost} method when it is disconnected.
"""
self.assertEqual(self.controller.connections, [self.proto])
self.proto.connectionLost(
Failure(ConnectionDone("Fake Connection Done")))
self.assertEqual(self.controller.connections, [])
def test_logoutOnError(self):
"""
The realm's logout callback is also invoked if there is an error
generating the response (for example, if the client disconnects
early).
"""
request = self._logoutTest()
request.processingFailed(
Failure(ConnectionDone("Simulated disconnect")))
self.assertEqual(self.realm.loggedOut, 1)
def test_connectionLost(self):
"""
If the connection is lost, all outstanding requests will errback.
"""
deferred1 = self.protocol.sendRequest("Admin", "Login")
deferred2 = self.protocol.sendRequest("Admin", "Login")
self.protocol.connectionLost(ConnectionDone())
failure1 = self.failureResultOf(deferred1)
failure1.trap(ConnectionDone)
failure2 = self.failureResultOf(deferred2)
failure2.trap(ConnectionDone)
def closed(self):
"""
When the channel closes, deliver disconnection notification to the
protocol.
"""
self._creator.cleanupConnection(self.conn, False)
if self._reason is None:
reason = ConnectionDone("ssh channel closed")
else:
reason = self._reason
self._protocol.connectionLost(Failure(reason))
def test_parametrizedClock(self):
"""
test_parametrizedClock
The clock used by L{KafkaBrokerClient} can be parametrized, so
that one can cleanly test reconnections.
"""
clock = Clock()
factory = KafkaBrokerClient('broker', reactor=clock)
factory.clientConnectionLost(FactoryAwareFakeConnector(None),
Failure(ConnectionDone()))
self.assertEqual(len(clock.calls), 2)
def test_connection_lost(self):
"""
``terminated`` fires when the connection is lost.
"""
self.test_receive_http_description()
self.protocol.connectionLost(Failure(ConnectionDone("done")))
self.assertIsInstance(
self.failureResultOf(self.terminated).value,
ConnectionDone,
)
def test_interruptedDeferredResponse(self):
"""
While waiting for the L{Deferred} returned by an L{XMLRPC} C{xmlrpc_*}
method to fire, the connection the request was issued over may close.
If this happens, neither C{write} nor C{finish} is called on the
request.
"""
self.resource.render(self.request)
self.request.processingFailed(failure.Failure(ConnectionDone("Simulated")))
self.result.callback("result")
self.assertEqual(self.request.written, [])
self.assertEqual(self.request.finished, 0)
def test_connectionLost_cleanly(self):
"""
The factory is notified of connection loss.
"""
kp = KafkaProtocol()
kp.factory = factory_spy = mock.Mock(wraps=TheFactory())
reason = Failure(ConnectionDone())
kp.connectionLost(reason)
factory_spy._connectionLost.assert_called_once_with(reason)
self.assertIsNone(kp.factory)
def test_parametrizedClock(self):
"""
The clock used by L{KafkaBrokerClient} can be parametrized, so
that one can cleanly test reconnections.
"""
reactor = MemoryReactorClock()
factory = KafkaBrokerClient(reactor, 'broker', 9092, 'clientId')
# XXX This ignores internal invariants, not a great test...
factory.clientConnectionLost(FactoryAwareFakeConnector(None),
Failure(ConnectionDone()))
self.assertEqual(len(reactor.getDelayedCalls()), 2)