def request(self, request):
if self._state != 'QUIESCENT':
return fail(RequestNotSent())
self._state = 'TRANSMITTING'
_requestDeferred = maybeDeferred(request.writeTo, self.transport)
self._finishedRequest = Deferred()
self._currentRequest = request
self._transportProxy = TransportProxyProducer(self.transport)
self._parser = HTTPClientParser(request, self._finishResponse)
self._parser.makeConnection(self._transportProxy)
self._responseDeferred = self._parser._responseDeferred
def cbRequestWrotten(ignored):
if self._state == 'TRANSMITTING':
self._state = 'WAITING'
self._responseDeferred.chainDeferred(self._finishedRequest)
def ebRequestWriting(err):
if self._state == 'TRANSMITTING':
self._state = 'GENERATION_FAILED'
self.transport.loseConnection()
self._finishedRequest.errback(
failure.Failure(RequestGenerationFailed([err])))
else:
log.err(
err, 'Error writing request, but not in valid state '
'to finalize request: %s' % self._state)
_requestDeferred.addCallbacks(cbRequestWrotten, ebRequestWriting)
return self._finishedRequest
def sendHandshake(self):
"""Perform a WebSocket opening handshake.
The protocol will use the L{Handshake} parameters of the
C{config} attribute of the factory.
Once the handshake is complete, the C{deferred} attribute
of the factory will be fired, either with C{None} (if the handshake
was successful) or with a failure (if the handshake failed).
"""
# The HTTP handshake request to send
request = self.handshake.buildRequest()
# We set a failing parser finisher because we take the transport
# control back as soon as all response headers are received, so we
# shouldn't ever deliver any data to the parser beside the headers.
def finisher(_):
raise RuntimeError("Parser cannot notify response finish")
# From this point on all data we received will be forwarded to the
# HTTP parser, see dataReceived,
self._parser = HTTPClientParser(request, finisher)
self._parser._responseDeferred.addCallback(self._handshakeResponse)
self._parser._responseDeferred.addErrback(self._handshakeFailure)
self._parser.makeConnection(self.transport)
request.writeTo(self.transport)
class HTTP11ClientProtocol(_newclient.HTTP11ClientProtocol):
def request(self, request):
if self._state != 'QUIESCENT':
return fail(RequestNotSent())
self._state = 'TRANSMITTING'
_requestDeferred = maybeDeferred(request.writeTo, self.transport)
self._finishedRequest = Deferred()
self._currentRequest = request
self._transportProxy = TransportProxyProducer(self.transport)
self._parser = HTTPClientParser(request, self._finishResponse)
self._parser.makeConnection(self._transportProxy)
self._responseDeferred = self._parser._responseDeferred
def cbRequestWrotten(ignored):
if self._state == 'TRANSMITTING':
self._state = 'WAITING'
self._responseDeferred.chainDeferred(self._finishedRequest)
def ebRequestWriting(err):
if self._state == 'TRANSMITTING':
self._state = 'GENERATION_FAILED'
self.transport.loseConnection()
self._finishedRequest.errback(
failure.Failure(RequestGenerationFailed([err])))
else:
log.err(err, 'Error writing request, but not in valid state '
'to finalize request: %s' % self._state)
_requestDeferred.addCallbacks(cbRequestWrotten, ebRequestWriting)
return self._finishedRequest
def connectionLost(self, reason): # NOQA: N802
if self.encoding:
self.data = self.data.decode(self.encoding).encode('utf8')
with open(self.agent.filename, 'wb') as _file:
_file.write(self.data)
self.finished.callback(self.agent.filename)
self.state = u'DONE'
HTTPClientParser.connectionLost(self, reason)
def request(self, request):
"""
Issue C{request} over C{self.transport} and return a L{Deferred} which
will fire with a L{Response} instance or an error.
@param request: The object defining the parameters of the request to
issue.
@type request: L{Request}
@rtype: L{Deferred}
@return: The deferred may errback with L{RequestGenerationFailed} if
the request was not fully written to the transport due to a local
error. It may errback with L{RequestTransmissionFailed} if it was
not fully written to the transport due to a network error. It may
errback with L{ResponseFailed} if the request was sent (not
necessarily received) but some or all of the response was lost. It
may errback with L{RequestNotSent} if it is not possible to send
any more requests using this L{HTTP11ClientProtocol}.
"""
if self._state != 'QUIESCENT':
return fail(RequestNotSent())
self._state = 'TRANSMITTING'
_requestDeferred = maybeDeferred(request.writeTo, self.transport)
self._finishedRequest = Deferred()
# Keep track of the Request object in case we need to call stopWriting
# on it.
self._currentRequest = request
self._transportProxy = TransportProxyProducer(self.transport)
self._parser = HTTPClientParser(request, self._finishResponse)
self._parser.makeConnection(self._transportProxy)
self._responseDeferred = self._parser._responseDeferred
def cbRequestWrotten(ignored):
if self._state == 'TRANSMITTING':
self._state = 'WAITING'
self._responseDeferred.chainDeferred(self._finishedRequest)
def ebRequestWriting(err):
if self._state == 'TRANSMITTING':
self._state = 'GENERATION_FAILED'
self.transport.loseConnection()
self._finishedRequest.errback(
Failure(RequestGenerationFailed([err])))
else:
log.err(err, "foo")
_requestDeferred.addCallbacks(cbRequestWrotten, ebRequestWriting)
return self._finishedRequest
class WebSocketsClientProtocol(WebSocketsProtocolWrapper, _FrameParser):
"""
Client-side version of WebSocketsProtocolWrapper, performing the handshake.
@ivar subProtocol: Once the handshake is complete, the sub-protocol that
the server has indicated us (or C{None} if not specified).
@type subProtocol: C{str}
@ivar handshake: Must be set by the factory, see L{WebSocketsClientFactory}
@type handshake: L{Handshake}.
@ivar deferred: A deferred which will callback once the opening handshake
has been successfully completed and the protocol is ready to send and
receive frames, or errback with a the relevant failure otherwise. Must
be set by the factory.
@type deferred: C{twisted.internet.defer.Deferred}
"""
isClient = True
subProtocol = None
handshake = None
deferred = None
_parser = None
def connectionMade(self):
self._buffer = []
self._receiver.makeConnection(_FrameSender(self.transport))
# Perform the handshake right after connecting.
self.sendHandshake()
def connectionLost(self, reason):
if self._parser is not None:
# This means we lost the connection before the handshake could
# be complete, let's errback.
self.deferred.errback(reason)
else:
# Forward the event to the user protocol
self.wrappedProtocol.connectionLost(reason)
def sendHandshake(self):
"""Perform a WebSocket opening handshake.
The protocol will use the L{Handshake} parameters of the
C{config} attribute of the factory.
Once the handshake is complete, the C{deferred} attribute
of the factory will be fired, either with C{None} (if the handshake
was successful) or with a failure (if the handshake failed).
"""
# The HTTP handshake request to send
request = self.handshake.buildRequest()
# We set a failing parser finisher because we take the transport
# control back as soon as all response headers are received, so we
# shouldn't ever deliver any data to the parser beside the headers.
def finisher(_):
raise RuntimeError("Parser cannot notify response finish")
# From this point on all data we received will be forwarded to the
# HTTP parser, see dataReceived,
self._parser = HTTPClientParser(request, finisher)
self._parser._responseDeferred.addCallback(self._handshakeResponse)
self._parser._responseDeferred.addErrback(self._handshakeFailure)
self._parser.makeConnection(self.transport)
request.writeTo(self.transport)
def handshakeMade(self):
"""Surrogate for connectionMade. Called after protocol negotiation."""
self.wrappedProtocol.makeConnection(self)
self.deferred.callback(self.wrappedProtocol)
def abortHandshake(self, reason):
"""Abort the handshake, propagating the given error.
@param reason: Details about the error.
@type reason: L{twisted.python.Failure}
"""
# Make sure we were performing the handshake
assert self._parser is not None, "No handshake in progress"
# This marks that the handshake has completed
self._parser = None
# Handshake errors are fatal, let's close the connection
self.transport.abortConnection()
self.deferred.errback(reason)
def dataReceived(self, data):
"""
Forward data to the HTTP parser if we are performing an handshake or
process it as WebSocket payload otherwise.
"""
if self._parser is not None:
try:
self._parser.dataReceived(data)
except:
# Disconnect the parser with the current failure, which will
# cause the handshake deferred to errback.
self._parser.connectionLost(Failure())
else:
WebSocketsProtocolWrapper.dataReceived(self, data)
def _handshakeResponse(self, response):
"""
Parse the HTTP response for the handshake request.
"""
# Check the status code
if response.code != 101:
raise HandshakeWrongStatus(response.code)
# Check the accept key
accept = response.headers.getRawHeaders("Sec-WebSocket-Accept")
if not accept or accept[0] != _makeAccept(self.handshake.key):
raise HandshakeWrongAcceptKey(self.handshake.key, accept)
# This marks that the handshake has completed
self._parser = None
# Figure out what protocol the server has chosen, if any
protocol = response.headers.getRawHeaders("Sec-WebSocket-Protocol")
if protocol:
self.subProtocol = protocol[0]
# The trasport was paused by the parser to avoid consuming non-header
# data. Let's resume it.
self.transport.resumeProducing()
self.handshakeMade()
def _handshakeFailure(self, failure):
"""
Pass on the failure to L{abortHandshake}.
"""
if failure.check(ResponseFailed):
# The parser failed to parse the response, let's wrap it in
# a HandshakeProtocolError
failure = Failure(HandshakeProtocolError(failure.value))
# Bail out in case of unexpected errors
failure.trap(HandshakeError)
self.abortHandshake(failure)
def _parseFrames(self):
_FrameParser._parseFrames(self)
class ZenHTTP11ClientProtocol(HTTP11ClientProtocol):
@property
def state(self):
return self._state
def request(self, request):
"""
Issue C{request} over C{self.transport} and return a L{Deferred} which
will fire with a L{Response} instance or an error.
@param request: The object defining the parameters of the request to
issue.
@type request: L{Request}
@rtype: L{Deferred}
@return: The deferred may errback with L{RequestGenerationFailed} if
the request was not fully written to the transport due to a local
error. It may errback with L{RequestTransmissionFailed} if it was
not fully written to the transport due to a network error. It may
errback with L{ResponseFailed} if the request was sent (not
necessarily received) but some or all of the response was lost. It
may errback with L{RequestNotSent} if it is not possible to send
any more requests using this L{HTTP11ClientProtocol}.
"""
if self._state != 'QUIESCENT':
return fail(RequestNotSent())
self._state = 'TRANSMITTING'
_requestDeferred = maybeDeferred(request.writeTo, self.transport)
self._finishedRequest = Deferred()
# Keep track of the Request object in case we need to call stopWriting
# on it.
self._currentRequest = request
self._transportProxy = TransportProxyProducer(self.transport)
self._parser = HTTPClientParser(request, self._finishResponse)
self._parser.makeConnection(self._transportProxy)
self._responseDeferred = self._parser._responseDeferred
def cbRequestWrotten(ignored):
if self._state == 'TRANSMITTING':
self._state = 'WAITING'
self._responseDeferred.chainDeferred(self._finishedRequest)
def ebRequestWriting(err):
if self._state == 'TRANSMITTING':
self._state = 'GENERATION_FAILED'
self.transport.loseConnection()
self._finishedRequest.errback(
Failure(RequestGenerationFailed([err])))
else:
log.err(err, "foo")
_requestDeferred.addCallbacks(cbRequestWrotten, ebRequestWriting)
return self._finishedRequest
def _finishResponse(self, rest):
"""
Called by an L{HTTPClientParser} to indicate that it has parsed a
complete response.
@param rest: A C{str} giving any trailing bytes which were given to
the L{HTTPClientParser} which were not part of the response it
was parsing.
"""
assert self._state in ('WAITING', 'TRANSMITTING')
if self._state == 'WAITING':
self._state = 'QUIESCENT'
else:
# The server sent the entire response before we could send the
# whole request. That sucks. Oh well. Fire the request()
# Deferred with the response. But first, make sure that if the
# request does ever finish being written that it won't try to fire
# that Deferred.
self._state = 'TRANSMITTING_AFTER_RECEIVING_RESPONSE'
self._responseDeferred.chainDeferred(self._finishedRequest)
reason = ConnectionDone("synthetic!")
connHeaders = self._parser.connHeaders.getRawHeaders('connection')
if (connHeaders is not None) and ('close' in connHeaders):
self._giveUp(Failure(reason))
else:
# It's persistent connection
self._disconnectParser(reason)
def allHeadersReceived(self):
HTTPClientParser.allHeadersReceived(self)
self.response.deliverBody(ProxyProtocol(self.forwardData))
def lineReceived(self, line):
""" Forwards the headers exactly as they arrive """
if line[-1:] == '\r':
line = line[:-1]
self.forwardData(line + '\r\n')
HTTPClientParser.lineReceived(self, line)
def fetch(self, path, list_as_set=False, **kwargs):
if type(kwargs.get('postdata', None)) == dict:
kwargs['postdata'] = urlencode(kwargs['postdata'], doseq=True)
if self.use_networking:
dResponse = cyclone.httpclient.fetch(
url=self.getUrl(path=path), **kwargs
)
else:
from cyclone.httpclient import Receiver
from twisted.web._newclient import Request, Response
from twisted.web.http_headers import Headers
from twisted.test.proto_helpers import AccumulatingProtocol
server_proto = self.app.buildProtocol( ('127.0.0.1', 0) )
server_transport = proto_helpers.StringTransport()
server_proto.makeConnection(server_transport)
rawHeaders = kwargs.get('headers', {})
rawHeaders['Host'] = ['localhost']
req = Request(
method=kwargs['method'],
uri=path,
headers=Headers( rawHeaders=rawHeaders ),
bodyProducer=None,
)
req.writeTo(server_transport)
server_proto.dataReceived(server_transport.value())
print server_transport.value()
# Strip out the original request.
parts = server_transport.value().split("\r\n\r\n")
actual_response = "\r\n\r\n".join(parts[1:])
print actual_response
from twisted.web._newclient import HTTPClientParser
client_parser = HTTPClientParser(req, lambda x:x)
client_parser.makeConnection(proto_helpers.StringTransport())
client_parser.dataReceived(actual_response)
response = client_parser.response
# This was done in cyclone.httpclient
if response.code in (204, 304):
response.body = ''
dResponse = defer.succeed(response)
else:
dResponse = defer.Deferred()
response.deliverBody(Receiver(dResponse))
def set_response(value):
response.body = value
return response
dResponse.addCallback(set_response)
# This triggers response.deliverBody. It needs a reason of some kind
client_parser.connectionLost('Finished')
if list_as_set:
def decode_body(res):
res.body = json.loads(res.body, object_hook=decode_list_as_set)
return res
dResponse.addCallback(decode_body)
else:
def decode_body(res):
try:
res.body = json.loads(res.body)
except:
print "counld not parse body: %s" % res.body
return res
dResponse.addCallback(decode_body)
return dResponse
