def finish(r): # By now the Agent should have all it needs to parse a response. protocol.connectionLost(Failure(ConnectionDone())) # Tell it that the connection is now complete so it can clean up. channel.connectionLost(Failure(ConnectionDone())) # Propogate the response. return r
def request(self, method, uri, headers=None, bodyProducer=None): """ Implement IAgent.request. """ # We want to use Agent to parse the HTTP response, so let's ask it to # make a request against our in-memory reactor. response = self._realAgent.request(method, uri, headers, bodyProducer) # That will try to establish an HTTP connection with the reactor's # connectTCP method, and MemoryReactor will place Agent's factory into # the tcpClients list. We'll extract that. host, port, factory, timeout, bindAddress = ( self._memoryReactor.tcpClients[0]) # Then we need to convince that factory it's connected to something and # it will give us a protocol for that connection. protocol = factory.buildProtocol(None) # We want to capture the output of that connection so we'll make an # in-memory transport. clientTransport = AbortableStringTransport() # When the protocol is connected to a transport, it ought to send the # whole request because callers of this should not use an asynchronous # bodyProducer. protocol.makeConnection(clientTransport) # Get the data from the request. requestData = clientTransport.io.getvalue() # Now time for the server to do its job. Ask it to build an HTTP # channel. channel = Site(self._rootResource).buildProtocol(None) # Connect the channel to another in-memory transport so we can collect # the response. serverTransport = StringTransport() serverTransport.hostAddr = IPv4Address('TCP', '127.0.0.1', 80) channel.makeConnection(serverTransport) # Feed it the data that the Agent synthesized. channel.dataReceived(requestData) # Tell it that the connection is now complete so it can clean up. channel.connectionLost(Failure(ConnectionDone())) # Now we have the response data, let's give it back to the Agent. protocol.dataReceived(serverTransport.io.getvalue()) # By now the Agent should have all it needs to parse a response. protocol.connectionLost(Failure(ConnectionDone())) # Return the response in the accepted format (Deferred firing # IResponse). This should be synchronously fired, and if not, it's the # system under test's problem. return response
def test_connectionLostWithoutAvatar(self): """ L{CredReceiver.connectionLost} does not raise an exception if no login has occurred when it is called. """ self.server.connectionLost( Failure(ConnectionDone("test connection lost")))
def test_connectionClosedBeforeSecure(self): """ If the connection closes at any point before the SSH transport layer has finished key exchange (ie, gotten to the point where we may attempt to authenticate), the L{Deferred} returned by L{SSHCommandClientEndpoint.connect} fires with a L{Failure} wrapping the reason for the lost connection. """ endpoint = SSHCommandClientEndpoint.newConnection( self.reactor, b"/bin/ls -l", b"dummy user", self.hostname, self.port, knownHosts=self.knownHosts, ui=FixedResponseUI(False)) factory = Factory() factory.protocol = Protocol d = endpoint.connect(factory) transport = StringTransport() factory = self.reactor.tcpClients[0][2] client = factory.buildProtocol(None) client.makeConnection(transport) client.connectionLost(Failure(ConnectionDone())) self.failureResultOf(d).trap(ConnectionDone)
def test_connectionCancelledBeforeSecure(self): """ If the connection is cancelled before the SSH transport layer has finished key exchange (ie, gotten to the point where we may attempt to authenticate), the L{Deferred} returned by L{SSHCommandClientEndpoint.connect} fires with a L{Failure} wrapping L{CancelledError} and the connection is aborted. """ endpoint = SSHCommandClientEndpoint.newConnection( self.reactor, b"/bin/ls -l", b"dummy user", self.hostname, self.port, knownHosts=self.knownHosts, ui=FixedResponseUI(False)) factory = Factory() factory.protocol = Protocol d = endpoint.connect(factory) transport = AbortableFakeTransport(None, isServer=False) factory = self.reactor.tcpClients[0][2] client = factory.buildProtocol(None) client.makeConnection(transport) d.cancel() self.failureResultOf(d).trap(CancelledError) self.assertTrue(transport.aborted) # Make sure the connection closing doesn't result in unexpected # behavior when due to cancellation: client.connectionLost(Failure(ConnectionDone()))
def frombytes(cls, byte_string): """ Parse an AWS request from a byte string. @param byte_string: The request as parsed from a C{.req} file. @type byte_string: L{bytes} @return: A request object. @rtype: L{_AWSRequest} """ # Ensure there's a blank line so that status and header # parsing completes. blank_line = b'\n\n' if blank_line not in byte_string: byte_string += blank_line channel = HTTPChannel() channel.delimiter = b'\n' channel.makeConnection(StringTransport()) channel.dataReceived(byte_string) channel.connectionLost(ConnectionDone()) request = channel.requests[-1] method = request.method path = request.uri headers = dict(request.requestHeaders.getAllRawHeaders()) # what comes after the empty line is a body = byte_string.split(blank_line, 1)[-1] return cls(method, path, headers, body)
def _finishResponse(self, rest): """ Called by an L{HTTPClientParser} to indicate that it has parsed a complete response. @param rest: A C{str} giving any trailing bytes which were given to the L{HTTPClientParser} which were not part of the response it was parsing. """ assert self._state in ('WAITING', 'TRANSMITTING') if self._state == 'WAITING': self._state = 'QUIESCENT' else: # The server sent the entire response before we could send the # whole request. That sucks. Oh well. Fire the request() # Deferred with the response. But first, make sure that if the # request does ever finish being written that it won't try to fire # that Deferred. self._state = 'TRANSMITTING_AFTER_RECEIVING_RESPONSE' self._responseDeferred.chainDeferred(self._finishedRequest) reason = ConnectionDone("synthetic!") connHeaders = self._parser.connHeaders.getRawHeaders('connection') if (connHeaders is not None) and ('close' in connHeaders): self._giveUp(Failure(reason)) else: # It's persistent connection self._disconnectParser(reason)
def test_kill_unexpected_exit(self): """ The :py:class:`Deferred` returned by :py:meth:`_HTTPBinProcess.kill` errbacks with the failure when it is not :py:class:`ProcessTerminated`, or its signal does not match the expected signal. """ for error in [ ProcessTerminated(1, signal=signal.SIGIO), ConnectionDone("Bye") ]: httpbin_process = parent._HTTPBinProcess(https=False) httpbin_process.server_description(self.reactor) spawned_process = self.reactor.spawnedProcesses[-1] spawned_process.send_stdout( shared._HTTPBinDescription(host="host", port=1234).to_json_bytes() + b'\n') termination_deferred = httpbin_process.kill() spawned_process.end_process(Failure(error)) self.assertIs( self.failureResultOf(termination_deferred).value, error)
def _disconnected(self, reason): """ Called when the stream has been closed. From this point on, the manager doesn't interact with the L{XmlStream} anymore and notifies each handler that the connection was lost by calling its C{connectionLost} method. """ self.xmlstream = None self._initialized = False # Twisted versions before 11.0 passed an XmlStream here. if not hasattr(reason, 'trap'): reason = failure.Failure(ConnectionDone()) # Notify all child services which implement # the IService interface for e in list(self): e.connectionLost(reason) # This errbacks all deferreds of iq's for which no response has # been received with a L{ConnectionLost} failure. Otherwise, the # deferreds will never be fired. iqDeferreds = self._iqDeferreds self._iqDeferreds = {} for d in iqDeferreds.itervalues(): d.errback(reason)
def test_connection_done(test_client, mock_treq): mock_treq.post.side_effect = ConnectionDone() with pytest.raises(ConnectionDone): yield test_client.request("POST", "/test/uri", SOME_JSON, is_retry=True)
def test_disconnected(self): """ If the transport is disconnected before the response is available, no ``RuntimeError`` is logged for finishing a disconnected request. """ result = Deferred() resource = StaticResource(result) d = render(resource, {}) resource._request.connectionLost(Failure(ConnectionDone())) result.callback(b"Some result") self.assertThat( d, failed( AfterPreprocessing( lambda reason: reason.type, Equals(ConnectionDone), ), ), ) # Since we're not a trial TestCase we don't have flushLoggedErrors. # The next best thing is to make sure any dangling Deferreds have been # garbage collected and then let the generic trial logic for failing # tests with logged errors kick in. gc.collect()
def test_connectionMadeLost(self): """ C{connectionMade} and C{connectionLost} are called on the protocol by the C{SerialPort}. """ # Serial port that doesn't actually connect to anything: class DummySerialPort(serialport.SerialPort): _serialFactory = DoNothing def _finishPortSetup(self): pass # override default win32 actions events = [] class SerialProtocol(Protocol): def connectionMade(self): events.append("connectionMade") def connectionLost(self, reason): events.append(("connectionLost", reason)) # Creation of port should result in connectionMade call: port = DummySerialPort(SerialProtocol(), "", reactor=DoNothing()) self.assertEqual(events, ["connectionMade"]) # Simulate reactor calling connectionLost on the SerialPort: f = Failure(ConnectionDone()) port.connectionLost(f) self.assertEqual(events, ["connectionMade", ("connectionLost", f)])
def _reader(self): """Wait forever for new chunks. Notify the user about the good ones through self._on_incoming_cb. If a response to a previously sent chunk is received, pop the corresponding response_cb from self.expected_responses and send the response there. """ error = Failure(ConnectionDone()) try: self.writer_job.link(self.reader_job) try: while self.state in ['CONNECTED', 'FLUSHING']: chunk = self.msrp.read_chunk() if chunk.method is None: # response self._handle_incoming_response(chunk) else: method = getattr(self, '_handle_incoming_%s' % chunk.method, None) if method is not None: method(chunk) else: response = make_response(chunk, 501, 'Method unknown') self.outgoing.send(OutgoingChunk(response)) except proc.LinkedExited: # writer has exited pass finally: self.writer_job.unlink(self.reader_job) self.writer_job.kill() self.logger.debug('reader: expecting responses only') delay = time() - self.last_expected_response if delay >= 0 and self.expected_responses: # continue read the responses until the last timeout expires with api.timeout(delay, None): while self.expected_responses: chunk = self.msrp.read_chunk() if chunk.method is None: self._handle_incoming_response(chunk) else: self.logger.debug('dropping incoming %r', chunk) # read whatever left in the queue with api.timeout(0, None): while self.msrp._queue: chunk = self.msrp.read_chunk() if chunk.method is None: self._handle_incoming_response(chunk) else: self.logger.debug('dropping incoming %r', chunk) self.logger.debug('reader: done') except ConnectionClosedErrors as e: self.logger.debug('reader: exiting because of %r', e) error = Failure(e) except Exception: self.logger.exception('reader: captured unhandled exception:') error = Failure() raise finally: self._on_incoming_cb(error=error) self.msrp.loseConnection(wait=False) self.set_state('DONE')
def test_stopTryingWhenConnected(self): """ If a L{KafkaBrokerClient} has C{stopTrying} called while it is connected, it does not subsequently attempt to reconnect if the connection is later lost. """ reactor = MemoryReactorClock() class NoConnectConnector(object): def stopConnecting(self): raise ClientError("Shouldn't be called, " "we're connected.") # pragma: no cover def connect(self): raise ClientError( "Shouldn't be reconnecting.") # pragma: no cover c = KafkaBrokerClient(reactor, 'broker', 9092, 'clientId') c.protocol = Protocol # Let's pretend we've connected: c.buildProtocol(None) # Now we stop trying, then disconnect: c.stopTrying() c.clientConnectionLost(NoConnectConnector(), Failure(ConnectionDone())) self.assertFalse(c.continueTrying)
def close(self, reason=None, within=0): """Explicitely close the connection. @param reason: Optional closing reason. If not given, ConnectionDone will be used. @param within: Shutdown the client within this amount of seconds. If zero (the default), all channels and queues will be closed immediately. If greater than 0, try to close the AMQP connection cleanly, by sending a "close" method and waiting for "close-ok". If no reply is received within the given amount of seconds, the transport will be forcely shutdown. """ if self.closed: return if reason is None: reason = ConnectionDone() if within > 0: channel0 = yield self.channel(0) deferred = channel0.connection_close() call = self.clock.callLater(within, deferred.cancel) try: yield deferred except defer.CancelledError: pass else: call.cancel() self.do_close(reason)
def test_writeConnectionLost(self): """ The protocol created by L{flowFountFromEndpoint} provides half-close support, and when it receives an EOF (i.e.: C{writeConnectionLost}) it will signal the end of the flow to its drain's fount, but not to its fount's drain. """ endpoint, ports = fakeEndpointWithPorts() fffep = flowFountFromEndpoint(endpoint) fffep.callback(None) flowFount = self.successResultOf(fffep) protocol = ports[0].factory.buildProtocol(None) verifyObject(IHalfCloseableProtocol, protocol) aTransport = StringTransport() protocol.makeConnection(aTransport) accepted = FakeDrain() flowFount.flowTo(accepted) [flow] = accepted.received receivedData = FakeDrain() dataSender = FakeFount() flow.fount.flowTo(receivedData) dataSender.flowTo(flow.drain) self.assertEqual(len(receivedData.stopped), 0) self.assertEqual(dataSender.flowIsStopped, False) protocol.writeConnectionLost() self.assertEqual(len(receivedData.stopped), 0) self.assertEqual(dataSender.flowIsStopped, 1) protocol.connectionLost(ConnectionDone()) self.assertEqual(len(receivedData.stopped), 1) self.assertEqual(dataSender.flowIsStopped, 1)
def test_process_exception_outside_crawlera(self): self.spider.crawlera_enabled = False crawler = self._mock_crawler(self.spider, self.settings) mw = self.mwcls.from_crawler(crawler) mw.open_spider(self.spider) req = Request("https://scrapy.org") assert mw.process_exception(req, ConnectionDone(), self.spider) is None
def test_connection_err(self): from twisted.internet.error import ConnectionDone try: raise ConnectionDone("Connection was closed cleanly.") except Exception: fail = Failure() self.base._response_err(fail) assert not self.status_mock.called
def test_connection_done(self): # pretend we connected self.proto._connectionMade() self.proto.connectionLost(Failure(ConnectionDone())) messages = ' '.join([str(x[1]) for x in self.proto.log.mock_calls]) self.assertTrue('closed cleanly' in messages)
def loseConnection(self): if not self.connected: return self.connected = False reason = Failure(ConnectionDone("Bye.")) self.protocol.connectionLost(reason) if self.connector is not None: self.connector.connectionLost(reason)
def test_requestDisconnectCleanup(self): """ Test if deferreds for iq's that haven't yet received a response have their errback called on stream disconnect. """ d = self.streamManager.request(self.request) xs = self.xmlstream xs.connectionLost(failure.Failure(ConnectionDone())) self.assertFailure(d, ConnectionDone) return d
def test_connectionTracking(self): """ L{dns.DNSProtocol} calls its controller's C{connectionMade} method with itself when it is connected to a transport and its controller's C{connectionLost} method when it is disconnected. """ self.assertEqual(self.controller.connections, [self.proto]) self.proto.connectionLost( Failure(ConnectionDone("Fake Connection Done"))) self.assertEqual(self.controller.connections, [])
def test_logoutOnError(self): """ The realm's logout callback is also invoked if there is an error generating the response (for example, if the client disconnects early). """ request = self._logoutTest() request.processingFailed( Failure(ConnectionDone("Simulated disconnect"))) self.assertEqual(self.realm.loggedOut, 1)
def test_connectionLost(self): """ If the connection is lost, all outstanding requests will errback. """ deferred1 = self.protocol.sendRequest("Admin", "Login") deferred2 = self.protocol.sendRequest("Admin", "Login") self.protocol.connectionLost(ConnectionDone()) failure1 = self.failureResultOf(deferred1) failure1.trap(ConnectionDone) failure2 = self.failureResultOf(deferred2) failure2.trap(ConnectionDone)
def closed(self): """ When the channel closes, deliver disconnection notification to the protocol. """ self._creator.cleanupConnection(self.conn, False) if self._reason is None: reason = ConnectionDone("ssh channel closed") else: reason = self._reason self._protocol.connectionLost(Failure(reason))
def test_parametrizedClock(self): """ test_parametrizedClock The clock used by L{KafkaBrokerClient} can be parametrized, so that one can cleanly test reconnections. """ clock = Clock() factory = KafkaBrokerClient('broker', reactor=clock) factory.clientConnectionLost(FactoryAwareFakeConnector(None), Failure(ConnectionDone())) self.assertEqual(len(clock.calls), 2)
def test_connection_lost(self): """ ``terminated`` fires when the connection is lost. """ self.test_receive_http_description() self.protocol.connectionLost(Failure(ConnectionDone("done"))) self.assertIsInstance( self.failureResultOf(self.terminated).value, ConnectionDone, )
def test_interruptedDeferredResponse(self): """ While waiting for the L{Deferred} returned by an L{XMLRPC} C{xmlrpc_*} method to fire, the connection the request was issued over may close. If this happens, neither C{write} nor C{finish} is called on the request. """ self.resource.render(self.request) self.request.processingFailed(failure.Failure(ConnectionDone("Simulated"))) self.result.callback("result") self.assertEqual(self.request.written, []) self.assertEqual(self.request.finished, 0)
def test_connectionLost_cleanly(self): """ The factory is notified of connection loss. """ kp = KafkaProtocol() kp.factory = factory_spy = mock.Mock(wraps=TheFactory()) reason = Failure(ConnectionDone()) kp.connectionLost(reason) factory_spy._connectionLost.assert_called_once_with(reason) self.assertIsNone(kp.factory)
def test_parametrizedClock(self): """ The clock used by L{KafkaBrokerClient} can be parametrized, so that one can cleanly test reconnections. """ reactor = MemoryReactorClock() factory = KafkaBrokerClient(reactor, 'broker', 9092, 'clientId') # XXX This ignores internal invariants, not a great test... factory.clientConnectionLost(FactoryAwareFakeConnector(None), Failure(ConnectionDone())) self.assertEqual(len(reactor.getDelayedCalls()), 2)