class StreamConnectFactory(SAMFactory):
protocol = StreamConnectProtocol
def __init__(self,
clientFactory,
session,
host,
dest,
port=None,
localPort=None):
self._clientFactory = clientFactory
self.session = session
self.host = host
self.dest = dest
self.port = port
self.localPort = localPort
self.deferred = Deferred(self._cancel)
def streamConnectionEstablished(self, streamProto):
self.session.addStream(streamProto)
peerAddress = I2PAddress(self.dest, self.host, self.port)
proto = self._clientFactory.buildProtocol(peerAddress)
if proto is None:
self.deferred.cancel()
return
streamProto.wrapProto(proto, peerAddress)
self.deferred.callback(proto)
def cancelled(*args, **kw):
"""
A function that takes any arguments at all and returns a Deferred
that is already cancelled.
"""
d = Deferred()
d.cancel()
return d
class BOBI2PClientFactory(ClientFactory):
protocol = I2PClientTunnelCreatorBOBClient
bobProto = None
canceled = False
removeTunnelWhenFinished = True
def _cancel(self, d):
self.bobProto.sender.transport.abortConnection()
self.canceled = True
def __init__(self, reactor, clientFactory, bobEndpoint, dest,
tunnelNick=None,
inhost='localhost',
inport=None,
options={}):
self._reactor = reactor
self._clientFactory = clientFactory
self._bobEndpoint = bobEndpoint
self.dest = dest
self.tunnelNick = tunnelNick
self.inhost = inhost
self.inport = inport
self.options = options
self.deferred = Deferred(self._cancel);
def buildProtocol(self, addr):
proto = self.protocol()
proto.factory = self
self.bobProto = proto
return proto
def bobConnectionFailed(self, reason):
if not self.canceled:
self.deferred.errback(reason)
# This method is not called if an endpoint deferred errbacks
def clientConnectionFailed(self, connector, reason):
self.bobConnectionFailed(reason)
def i2pTunnelCreated(self):
# BOB is now listening for a tunnel.
# BOB only listens on TCP4 (for now).
clientEndpoint = TCP4ClientEndpoint(self._reactor, self.inhost, self.inport)
# Wrap the client Factory.
wrappedFactory = BOBClientFactoryWrapper(self._clientFactory,
self._bobEndpoint,
I2PAddress(self.localDest),
self.tunnelNick,
self.removeTunnelWhenFinished)
wrappedFactory.setDest(self.dest)
d = clientEndpoint.connect(wrappedFactory)
def checkProto(proto):
if proto is None:
self.deferred.cancel()
return proto
d.addCallback(checkProto)
# When the Deferred returns an IProtocol, pass it on.
d.chainDeferred(self.deferred)
class BOBI2PClientFactory(ClientFactory):
protocol = I2PClientTunnelCreatorBOBClient
bobProto = None
canceled = False
removeTunnelWhenFinished = True
def _cancel(self, d):
self.bobProto.sender.transport.abortConnection()
self.canceled = True
def __init__(self, reactor, clientFactory, bobEndpoint, dest,
tunnelNick=None,
inhost='localhost',
inport=None,
options=None):
self._reactor = reactor
self._clientFactory = clientFactory
self._bobEndpoint = bobEndpoint
self.dest = dest
self.tunnelNick = tunnelNick
self.inhost = inhost
self.inport = inport
self.options = options
self.deferred = Deferred(self._cancel);
def buildProtocol(self, addr):
proto = self.protocol()
proto.factory = self
self.bobProto = proto
return proto
def bobConnectionFailed(self, reason):
if not self.canceled:
self.deferred.errback(reason)
# This method is not called if an endpoint deferred errbacks
def clientConnectionFailed(self, connector, reason):
self.bobConnectionFailed(reason)
def i2pTunnelCreated(self):
# BOB is now listening for a tunnel.
# BOB only listens on TCP4 (for now).
clientEndpoint = TCP4ClientEndpoint(self._reactor, self.inhost, self.inport)
# Wrap the client Factory.
wrappedFactory = BOBClientFactoryWrapper(self._clientFactory,
self._bobEndpoint,
I2PAddress(self.localDest),
self.tunnelNick,
self.removeTunnelWhenFinished)
wrappedFactory.setDest(self.dest)
d = clientEndpoint.connect(wrappedFactory)
def checkProto(proto):
if proto is None:
self.deferred.cancel()
return proto
d.addCallback(checkProto)
# When the Deferred returns an IProtocol, pass it on.
d.chainDeferred(self.deferred)
class HTTP11ClientProtocolPatched(HTTP11ClientProtocol):
def request(self, request):
if self._state != 'QUIESCENT':
return fail(RequestNotSent())
self._state = 'TRANSMITTING'
_requestDeferred = maybeDeferred(request.writeTo, self.transport)
def cancelRequest(ign):
if self.timedOut:
# Request timeout was hit
self._giveUp(Failure(TimeoutError("Request took longer than %s seconds" % request.timeout)))
else:
# Explicitly cancel the request's deferred if it's still trying to
# write when this request is cancelled.
if self._state in ('TRANSMITTING', 'TRANSMITTING_AFTER_RECEIVING_RESPONSE'):
_requestDeferred.cancel()
else:
self._giveUp(Failure(CancelledError()))
self._finishedRequest = Deferred(cancelRequest)
if self.timeout > 0:
self.timedOut = False
def timeout():
self.timedOut = True
self._finishedRequest.cancel()
td = reactor.callLater(self.timeout, timeout)
#td.cancel()
#self._finishedRequest.addBoth(td.cancel) # where/how to cancel?
# Keep track of the Request object in case we need to call stopWriting
# on it.
self._currentRequest = request
self._transportProxy = TransportProxyProducer(self.transport)
self._parser = HTTPClientParser(request, self._finishResponse)
self._parser.makeConnection(self._transportProxy)
self._responseDeferred = self._parser._responseDeferred
def cbRequestWrotten(ignored):
if self._state == 'TRANSMITTING':
self._state = 'WAITING'
self._responseDeferred.chainDeferred(self._finishedRequest)
def ebRequestWriting(err):
if self._state == 'TRANSMITTING':
self._state = 'GENERATION_FAILED'
self.transport.loseConnection()
self._finishedRequest.errback(
Failure(RequestGenerationFailed([err])))
else:
log.err(err, 'Error writing request, but not in valid state '
'to finalize request: %s' % self._state)
_requestDeferred.addCallbacks(cbRequestWrotten, ebRequestWriting)
return self._finishedRequest
def testControlReadvarExchangeEmptyStatusSet(self):
def final(result):
self.assertEqual(self.protocol.status, STATE_UNKNOWN)
d = Deferred()
d.addBoth(final)
self.protocol.d = d
self.protocol.controlReadvarExchange()
d.cancel()
return d
class MessagePackOobMethodContext(object):
__slots__ = 'd'
def __init__(self):
if Deferred is not None:
self.d = Deferred()
else:
self.d = None
def close(self):
if self.d is not None and not self.d.called:
self.d.cancel()
def testControlReadvarExchangePop(self):
def final(result):
self.assertFalse(self.protocol.peersToCheck)
d = Deferred()
d.addBoth(final)
self.protocol.d = d
self.protocol.peersToCheck = self.peersToCheck
self.protocol.minPeerSource = 100 # wrong value (too high)
self.protocol.controlReadvarExchange()
d.cancel()
class AsyncHttpJsonClient(object):
'''
Supports json request payload on both HTTP GET and POST
'''
def __init__(self, **kwargs):
# uri, method, body
for k, v in kwargs.items():
setattr(self, k, v)
if not kwargs.has_key('body'):
self.body = None
else:
self.body = JsonBodyProducer(self.body)
if not kwargs.has_key('method'):
self.method = 'GET'
if not kwargs.has_key('connectTimeout'):
self.connectTimeout = 3.0
self.agent = Agent(reactor, connectTimeout=self.connectTimeout)
self.__d_agent = self.agent.request(
self.method, self.uri,
Headers({
'User-Agent': ['AsyncHttpJsonRequest'],
'Content-Type': ['application/json'],
'Accept-Encoding': ['gzip']
}), self.body)
self.__deferredResponse = Deferred()
def __readResponseCallback(self, response, userCb, *cbargs):
response.deliverBody(
AsyncHttpResponseProtocol(self.__deferredResponse,
response.headers))
self.__deferredResponse.addCallback(userCb,
*([response] + list(cbargs)))
return self.__deferredResponse
def __readErrorCallback(self, error, userCb, *cbargs):
self.__deferredResponse.addErrback(userCb, *cbargs)
def addResponseCallback(self, callback, *cbargs):
self.__d_agent.addCallback(self.__readResponseCallback, callback,
*cbargs)
def addResponseErrback(self, callback, *cbargs):
self.__d_agent.addErrback(self.__readErrorCallback, callback, *cbargs)
def cancelRequest(self):
try:
self.__deferredResponse.cancel()
self.__d_agent.cancel()
except Exception, e:
logger.debug(str(e))
class AsyncHttpJsonClient(object):
'''
Supports json request payload on both HTTP GET and POST
'''
def __init__(self, **kwargs):
# uri, method, body
for k,v in kwargs.items():
setattr(self, k, v)
if not kwargs.has_key('body'):
self.body = None
else:
self.body = JsonBodyProducer(self.body)
if not kwargs.has_key('method'):
self.method = 'GET'
if not kwargs.has_key('connectTimeout'):
self.connectTimeout = 3.0
self.agent = Agent(reactor, connectTimeout=self.connectTimeout)
self.__d_agent = self.agent.request(
self.method,
self.uri,
Headers({
'User-Agent': ['AsyncHttpJsonRequest'],
'Content-Type': ['application/json'],
'Accept-Encoding': ['gzip']
}),
self.body)
self.__deferredResponse = Deferred()
def __readResponseCallback(self, response, userCb, *cbargs):
response.deliverBody(AsyncHttpResponseProtocol(self.__deferredResponse, response.headers))
self.__deferredResponse.addCallback(userCb, *([response]+list(cbargs)))
return self.__deferredResponse
def __readErrorCallback(self, error, userCb, *cbargs):
self.__deferredResponse.addErrback(userCb, *cbargs)
def addResponseCallback(self, callback, *cbargs):
self.__d_agent.addCallback(self.__readResponseCallback, callback, *cbargs)
def addResponseErrback(self, callback, *cbargs):
self.__d_agent.addErrback(self.__readErrorCallback, callback, *cbargs)
def cancelRequest(self):
try:
self.__deferredResponse.cancel()
self.__d_agent.cancel()
except Exception,e:
logger.debug(str(e))
def testSendReadStatRequestSent(self):
def final(err):
# err is a Failure instance because of d.cancel()
if self.protocol.timeoutCall.active():
self.protocol.timeoutCall.cancel()
self.assertTrue(self.protocol.transport.written)
d = Deferred()
d.addBoth(final)
self.protocol.d = d
self.protocol.sendReadstatRequest()
d.cancel()
return d
def testStartProtocolConnected(self):
def final(err):
# err is a Failure instance because of d.cancel()
if self.protocol.timeoutCall.active():
self.protocol.timeoutCall.cancel()
self.assertTrue(self.protocol.transport.connected)
d = Deferred()
d.addBoth(final)
self.protocol.d = d
self.protocol.startProtocol()
d.cancel()
return d
def testControlReadvarExchangePeerSet(self):
def final(err):
# err is a Failure instance because of d.cancel()
if self.protocol.timeoutCall.active():
self.protocol.timeoutCall.cancel()
self.assertEqual(self.protocol.currentPeer, self.peer)
d = Deferred()
d.addBoth(final)
self.protocol.d = d
self.protocol.peersToCheck = self.peersToCheck
self.protocol.controlReadvarExchange()
d.cancel()
return d
def testSendReadStatRequestCorrect(self):
def final(err):
# err is a Failure instance because of d.cancel()
if self.protocol.timeoutCall.active():
self.protocol.timeoutCall.cancel()
written_data = self.protocol.transport.written[-1][0]
expected_data = '\x16\x01\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00'
self.assertEqual(written_data, expected_data)
d = Deferred()
d.addBoth(final)
self.protocol.d = d
self.protocol.sendReadstatRequest()
d.cancel()
return d
def test_cancelledDeferred(self):
app = self.app
request = requestMock(b"/")
inner_d = Deferred()
@app.route("/")
def root(request):
return inner_d
d = _render(self.kr, request)
inner_d.cancel()
self.assertFired(d)
self.flushLoggedErrors(CancelledError)
def test_cancelledDeferred(self):
app = self.app
request = requestMock(b"/")
inner_d = Deferred()
@app.route("/")
def root(request):
return inner_d
d = _render(self.kr, request)
inner_d.cancel()
self.assertFired(d)
self.flushLoggedErrors(CancelledError)
class ConsumerQueue(object):
def __init__(self, stop_on_error=False, empty=None):
self.stop_on_error = stop_on_error
self.empty = empty
self.queue = DeferredQueue()
self.size = 0
self.running = True
self._deferred = Deferred()
def _consume_next(self, *args):
if not self.running:
return
self._deferred = self.queue.get()
self._deferred.addCallbacks(self._consumer, self._error)
def _consumer(self, item):
self.size -= 1
r = self.consume(item)
if self.size == 0 and self.empty is not None:
self.empty()
if isinstance(r, Deferred):
r.addCallbacks(self._consume_next, self._consume_next)
else:
self._consume_next()
def _error(self, fail):
self.error(fail)
if not self.stop_on_error:
self._consume_next()
def add(self, item):
self.size += 1
self.queue.put(item)
def consume(self, item):
raise NotImplementedError
def error(self, fail):
raise NotImplementedError
def start(self):
self.running = True
self._consume_next()
def stop(self):
self.running = False
self._deferred.cancel()
def cancel(self):
# to signal _inlineCallbacks to not fire self.errback with CancelledError;
# otherwise we'd have to call `Deferred.cancel(self)` immediately, but it
# would be semantically unnice if, by the time the coroutine is told to do
# its clean-up routine, the inner Deferred hadn't yet actually been cancelled.
self.cancelling = True
# the _swallow_cancelled_error errback is added as the last one, so anybody else who is already listening for
# CancelledError will still get it.
if self.depends_on:
self.depends_on.addErrback(_swallow_cancelled_error)
self.depends_on.cancel()
del self.depends_on
self.addErrback(_swallow_cancelled_error)
Deferred.cancel(self)
def cancel(self):
# to signal _inlineCallbacks to not fire self.errback with CancelledError;
# otherwise we'd have to call `Deferred.cancel(self)` immediately, but it
# would be semantically unnice if, by the time the coroutine is told to do
# its clean-up routine, the inner Deferred hadn't yet actually been cancelled.
self.cancelling = True
# the _swallow_cancelled_error errback is added as the last one, so anybody else who is already listening for
# CancelledError will still get it.
if self.depends_on:
self.depends_on.addErrback(_swallow_cancelled_error)
self.depends_on.cancel()
del self.depends_on
self.addErrback(_swallow_cancelled_error)
Deferred.cancel(self)
def testSendReadvarRequestCorrect(self):
def final(err):
# err is a Failure instance because of d.cancel()
if self.protocol.timeoutCall.active():
self.protocol.timeoutCall.cancel()
written_data = self.protocol.transport.written[-1][0]
expected_data = '\x16\x02\x00\x02\x00\x00g\xf3\x00\x00\x00\x06offset\x00\x00\x00\x00\x00\x00'
self.assertEqual(written_data, expected_data)
d = Deferred()
d.addBoth(final)
self.protocol.d = d
self.protocol.currentPeer = self.peer
self.protocol.sequenceCounter = 2
self.protocol.sendReadvarRequest()
d.cancel()
return d
def test_cancelledDeferred(self):
app = self.app
request = requestMock("/")
inner_d = Deferred()
@app.route("/")
def root(request):
return inner_d
d = _render(self.kr, request)
inner_d.cancel()
def _cb(result):
self.assertIdentical(result, None)
self.flushLoggedErrors(CancelledError)
d.addCallback(_cb)
return d
class StreamAcceptFactory(SAMFactory):
protocol = StreamAcceptProtocol
def __init__(self, clientFactory, session, listeningPort):
self._clientFactory = clientFactory
self.session = session
self.listeningPort = listeningPort
self.deferred = Deferred(self._cancel)
def streamAcceptEstablished(self, streamProto):
self.session.addStream(streamProto)
self.listeningPort.addAccept(streamProto)
def streamAcceptIncoming(self, streamProto):
self.listeningPort.removeAccept(streamProto)
proto = self._clientFactory.buildProtocol(streamProto.peer)
if proto is None:
self.deferred.cancel()
return
streamProto.wrapProto(proto, streamProto.peer, True)
class StreamAcceptFactory(SAMFactory):
protocol = StreamAcceptProtocol
def __init__(self, clientFactory, session, listeningPort):
self._clientFactory = clientFactory
self.session = session
self.listeningPort = listeningPort
self.deferred = Deferred(self._cancel);
def streamAcceptEstablished(self, streamProto):
self.session.addStream(streamProto)
self.listeningPort.addAccept(streamProto)
def streamAcceptIncoming(self, streamProto):
self.listeningPort.removeAccept(streamProto)
proto = self._clientFactory.buildProtocol(streamProto.peer)
if proto is None:
self.deferred.cancel()
return
streamProto.wrapProto(proto, streamProto.peer)
class StreamConnectFactory(SAMFactory):
protocol = StreamConnectProtocol
def __init__(self, clientFactory, session, host, dest, port=None, localPort=None):
self._clientFactory = clientFactory
self.session = session
self.host = host
self.dest = dest
self.port = port
self.localPort = localPort
self.deferred = Deferred(self._cancel);
def streamConnectionEstablished(self, streamProto):
self.session.addStream(streamProto)
peerAddress = I2PAddress(self.dest, self.host, self.port)
proto = self._clientFactory.buildProtocol(peerAddress)
if proto is None:
self.deferred.cancel()
return
streamProto.wrapProto(proto, peerAddress)
self.deferred.callback(proto)
def fetch_next(self, wait=True):
timeout = Cursor._wait_to_timeout(wait)
deadline = None if timeout is None else time.time() + timeout
def wait_canceller(d):
d.errback(ReqlTimeoutError())
while len(self.items) == 0 and self.error is None:
self._maybe_fetch_batch()
wait = Deferred(canceller=wait_canceller)
self.waiting.append(wait)
if deadline is not None:
timeout = max(0, deadline - time.time())
reactor.callLater(timeout, lambda: wait.cancel())
yield wait
returnValue(not self._is_empty() or self._has_error())
def fetch_next(self, wait=True):
timeout = Cursor._wait_to_timeout(wait)
deadline = None if timeout is None else time.time() + timeout
def wait_canceller(d):
d.errback(ReqlTimeoutError())
while len(self.items) == 0 and self.error is None:
self._maybe_fetch_batch()
wait = Deferred(canceller=wait_canceller)
self.waiting.append(wait)
if deadline is not None:
timeout = max(0, deadline - time.time())
reactor.callLater(timeout, lambda: wait.cancel())
yield wait
returnValue(not self._is_empty() or self._has_error())
class BOBI2PServerFactory(Factory):
protocol = I2PServerTunnelCreatorBOBClient
bobProto = None
canceled = False
removeTunnelWhenFinished = True
def _cancel(self, d):
self.bobProto.sender.transport.abortConnection()
self.canceled = True
def __init__(self,
reactor,
serverFactory,
bobEndpoint,
keyfile,
tunnelNick=None,
outhost='localhost',
outport=None,
options={}):
self._reactor = reactor
self._serverFactory = serverFactory
self._bobEndpoint = bobEndpoint
self._keyfile = keyfile
self._writeKeypair = False
self.tunnelNick = tunnelNick
self.outhost = outhost
self.outport = outport
self.options = options
self.deferred = Deferred(self._cancel)
def startFactory(self):
try:
f = open(self._keyfile, 'r')
self.keypair = f.read()
f.close()
except IOError:
self.keypair = None
self._writeKeypair = True
def buildProtocol(self, addr):
proto = self.protocol()
proto.factory = self
self.bobProto = proto
return proto
def bobConnectionFailed(self, reason):
if not self.canceled:
self.deferred.errback(reason)
# This method is not called if an endpoint deferred errbacks
def clientConnectionFailed(self, connector, reason):
self.bobConnectionFailed(reason)
def i2pTunnelCreated(self):
if self._writeKeypair:
try:
f = open(self._keyfile, 'w')
f.write(self.keypair)
f.close()
except IOError:
print 'Could not save keypair'
# BOB will now forward data to a listener.
# BOB only forwards to TCP4 (for now).
serverEndpoint = TCP4ServerEndpoint(self._reactor, self.outport)
# Wrap the server Factory.
wrappedFactory = BOBServerFactoryWrapper(self._serverFactory,
self._bobEndpoint,
I2PAddress(self.localDest),
self.tunnelNick,
self.removeTunnelWhenFinished)
d = serverEndpoint.listen(wrappedFactory)
def handlePort(port):
if port is None:
self.deferred.cancel()
serverAddr = I2PAddress(self.localDest)
p = I2PListeningPort(port, wrappedFactory, serverAddr)
return p
d.addCallback(handlePort)
# When the Deferred returns an IListeningPort, pass it on.
d.chainDeferred(self.deferred)
class BaseWrapper(RO.AddCallback.BaseMixin):
"""!A wrapper for a client talking to a server
This wrapper is responsible for starting and stopping everything:
- It accepts an actor wrapper
- It builds an actor dispatcher when the actor wrapper is ready
- It stops both on close()
Public attributes include:
- actorWrapper: the actor wrapper (twistedActor.ActorWrapper)
- dispatcher: the actor dispatcher (twistedActor.ActorDispatcher); None until ready
- readyDeferred: called when the dispatcher is ready
(for tracking closure use the Deferred returned by the close method, or stateCallback).
Subclasses must override:
_basicClose
isDone
isReady,
didFail
and must call _stateChanged as appropriate; see, e.g. DeviceWrapper
"""
def __init__(
self,
name="",
stateCallback=None,
callNow=False,
debug=False,
):
"""!Construct a DispatcherWrapper that manages everything
@param[in] name a name to use for messages
@param[in] stateCallback function to call when connection state of of any socket changes;
receives one argument: this actor wrapper
@param[in] callNow call stateCallback now? (Defaults to false because typically
subclasses have some additional setup to do before calling callback functions).
@param[in] debug print debug messages to stdout?
"""
self.name = name
RO.AddCallback.BaseMixin.__init__(self, defCallNow=True)
self.debug = bool(debug)
self.readyDeferred = Deferred()
self._closeDeferred = None
self.addCallback(stateCallback, callNow=callNow)
@property
def isReady(self):
"""!Return True if the actor has connected to the fake hardware controller
"""
raise NotImplementedError()
@property
def isDone(self):
"""!Return True if the actor and fake hardware controller are fully disconnected
"""
raise NotImplementedError()
@property
def didFail(self):
"""!Return True if isDone and there is a failure
"""
return self.isDone and self.isFailing
@property
def isFailing(self):
"""!Return True if there is a failure
"""
raise NotImplementedError()
def debugMsg(self, msgStr):
if self.debug:
print("%s: %s" % (self, msgStr))
def _basicClose(self):
"""!Close clients and servers
"""
raise NotImplementedError()
def _stateChanged(self, *args):
"""!Called when state changes
"""
self.debugMsg("_stateChanged(): isReady=%s, isDone=%s, didFail=%s, isFailing=%s, _closeDeferred=%s" % \
(self.isReady, self.isDone, self.didFail, self.isFailing, str(self._closeDeferred)))
if self.isFailing and not self.isDone and not self._closeDeferred:
self.close()
return
if self._closeDeferred: # closing or closed
if self.isDone:
if not self.readyDeferred.called:
self.debugMsg(
"canceling readyDeferred in _stateChanged; this should not happen"
)
self.readyDeferred.cancel()
if not self._closeDeferred.called:
self.debugMsg("calling closeDeferred")
self._closeDeferred.callback(None)
else:
sys.stderr.write(
"%s state changed after wrapper closed\n" % (self, ))
else: # opening or open
if not self.readyDeferred.called:
if self.isReady:
self.debugMsg("calling readyDeferred")
self.readyDeferred.callback(None)
elif self.didFail:
self.debugMsg("failing readyDeferred")
self.readyDeferred.errback(RuntimeError("failed"))
self._doCallbacks()
if self.isDone:
self._removeAllCallbacks()
def close(self):
"""!Close everything
@return a deferred
"""
self.debugMsg("close()")
if self._closeDeferred:
raise RuntimeError("Already closing or closed")
self._closeDeferred = Deferred()
if not self.readyDeferred.called:
self.debugMsg("canceling readyDeferred")
self.readyDeferred.cancel()
self._basicClose()
return self._closeDeferred
def __str__(self):
return "%s(%s)" % (type(self).__name__, self.name)
def __repr__(self):
return "%s(%s); isReady=%s, isDone=%s, didFail=%s, isFailing=%s" % \
(type(self).__name__, self.name, self.isReady, self.isDone, self.didFail, self.isFailing)
class BOBI2PServerFactory(Factory):
protocol = I2PServerTunnelCreatorBOBClient
bobProto = None
canceled = False
removeTunnelWhenFinished = True
def _cancel(self, d):
self.bobProto.sender.transport.abortConnection()
self.canceled = True
def __init__(self, reactor, serverFactory, bobEndpoint, keypairPath,
tunnelNick=None,
outhost='localhost',
outport=None,
options=None):
self._reactor = reactor
self._serverFactory = serverFactory
self._bobEndpoint = bobEndpoint
self._keypairPath = keypairPath
self._writeKeypair = False
self.tunnelNick = tunnelNick
self.outhost = outhost
self.outport = outport
self.options = options
self.deferred = Deferred(self._cancel)
def startFactory(self):
try:
keypairFile = open(self._keypairPath, 'r')
self.keypair = keypairFile.read()
keypairFile.close()
except IOError:
self.keypair = None
self._writeKeypair = True
def buildProtocol(self, addr):
proto = self.protocol()
proto.factory = self
self.bobProto = proto
return proto
def bobConnectionFailed(self, reason):
if not self.canceled:
self.deferred.errback(reason)
# This method is not called if an endpoint deferred errbacks
def clientConnectionFailed(self, connector, reason):
self.bobConnectionFailed(reason)
def i2pTunnelCreated(self):
if self._writeKeypair:
try:
keypairFile = open(self._keypairPath, 'w')
keypairFile.write(self.keypair)
keypairFile.close()
except IOError:
print 'Could not save keypair'
# BOB will now forward data to a listener.
# BOB only forwards to TCP4 (for now).
serverEndpoint = TCP4ServerEndpoint(self._reactor, self.outport)
# Wrap the server Factory.
wrappedFactory = BOBServerFactoryWrapper(self._serverFactory,
self._bobEndpoint,
I2PAddress(self.localDest),
self.tunnelNick,
self.removeTunnelWhenFinished)
d = serverEndpoint.listen(wrappedFactory)
def handlePort(port):
if port is None:
self.deferred.cancel()
serverAddr = I2PAddress(self.localDest)
p = I2PListeningPort(port, wrappedFactory, serverAddr)
return p
d.addCallback(handlePort)
# When the Deferred returns an IListeningPort, pass it on.
d.chainDeferred(self.deferred)
def root(request):
_d = Deferred()
request.notifyFinish().addErrback(lambda _: _d.cancel())
return _d
class BaseWrapper(RO.AddCallback.BaseMixin):
"""!A wrapper for a client talking to a server
This wrapper is responsible for starting and stopping everything:
- It accepts an actor wrapper
- It builds an actor dispatcher when the actor wrapper is ready
- It stops both on close()
Public attributes include:
- actorWrapper: the actor wrapper (twistedActor.ActorWrapper)
- dispatcher: the actor dispatcher (twistedActor.ActorDispatcher); None until ready
- readyDeferred: called when the dispatcher is ready
(for tracking closure use the Deferred returned by the close method, or stateCallback).
Subclasses must override:
_basicClose
isDone
isReady,
didFail
and must call _stateChanged as appropriate; see, e.g. DeviceWrapper
"""
def __init__(self,
name="",
stateCallback=None,
callNow=False,
debug=False,
):
"""!Construct a DispatcherWrapper that manages everything
@param[in] name a name to use for messages
@param[in] stateCallback function to call when connection state of of any socket changes;
receives one argument: this actor wrapper
@param[in] callNow call stateCallback now? (Defaults to false because typically
subclasses have some additional setup to do before calling callback functions).
@param[in] debug print debug messages to stdout?
"""
self.name = name
RO.AddCallback.BaseMixin.__init__(self, defCallNow=True)
self.debug = bool(debug)
self.readyDeferred = Deferred()
self._closeDeferred = None
self.addCallback(stateCallback, callNow=callNow)
@property
def isReady(self):
"""!Return True if the actor has connected to the fake hardware controller
"""
raise NotImplementedError()
@property
def isDone(self):
"""!Return True if the actor and fake hardware controller are fully disconnected
"""
raise NotImplementedError()
@property
def didFail(self):
"""!Return True if isDone and there is a failure
"""
return self.isDone and self.isFailing
@property
def isFailing(self):
"""!Return True if there is a failure
"""
raise NotImplementedError()
def debugMsg(self, msgStr):
if self.debug:
print("%s: %s" % (self, msgStr))
def _basicClose(self):
"""!Close clients and servers
"""
raise NotImplementedError()
def _stateChanged(self, *args):
"""!Called when state changes
"""
self.debugMsg("_stateChanged(): isReady=%s, isDone=%s, didFail=%s, isFailing=%s, _closeDeferred=%s" % \
(self.isReady, self.isDone, self.didFail, self.isFailing, str(self._closeDeferred)))
if self.isFailing and not self.isDone and not self._closeDeferred:
self.close()
return
if self._closeDeferred: # closing or closed
if self.isDone:
if not self.readyDeferred.called:
self.debugMsg("canceling readyDeferred in _stateChanged; this should not happen")
self.readyDeferred.cancel()
if not self._closeDeferred.called:
self.debugMsg("calling closeDeferred")
self._closeDeferred.callback(None)
else:
sys.stderr.write("%s state changed after wrapper closed\n" % (self,))
else: # opening or open
if not self.readyDeferred.called:
if self.isReady:
self.debugMsg("calling readyDeferred")
self.readyDeferred.callback(None)
elif self.didFail:
self.debugMsg("failing readyDeferred")
self.readyDeferred.errback(RuntimeError("failed"))
self._doCallbacks()
if self.isDone:
self._removeAllCallbacks()
def close(self):
"""!Close everything
@return a deferred
"""
self.debugMsg("close()")
if self._closeDeferred:
raise RuntimeError("Already closing or closed")
self._closeDeferred = Deferred()
if not self.readyDeferred.called:
self.debugMsg("canceling readyDeferred")
self.readyDeferred.cancel()
self._basicClose()
return self._closeDeferred
def __str__(self):
return "%s(%s)" % (type(self).__name__, self.name)
def __repr__(self):
return "%s(%s); isReady=%s, isDone=%s, didFail=%s, isFailing=%s" % \
(type(self).__name__, self.name, self.isReady, self.isDone, self.didFail, self.isFailing)
def root(request):
_d = Deferred()
_d.addErrback(cancelled.append)
request.notifyFinish().addCallback(lambda _: _d.cancel())
return _d
def root(request):
_d = Deferred()
request.notifyFinish().addErrback(lambda _: _d.cancel())
return _d
class Task():
"""
A class to represent a ``Task`` - a path to be backed up as soon as other backups will be finished.
All tasks are stored in the list, see ``tasks()`` method.
As soon as task get executed it fires the result call back and removed from the list.
When task executes a new backup job gets created.
"""
def __init__(self, pathID, localPath=None, keyID=None):
self.number = NewTaskNumber() # index number for the task
self.created = time.time()
self.backupID = None
self.pathID = None
self.fullGlobPath = None
self.fullCustomerID = None
self.customerGlobID = None
self.customerIDURL = None
self.remotePath = None
self.keyID = None
self.keyAlias = None
self.result_defer = Deferred()
self.result_defer.addCallback(OnTaskExecutedCallback)
self.result_defer.addErrback(OnTaskFailedCallback)
parts = self.set_path_id(pathID)
self.set_key_id(keyID or my_keys.make_key_id(alias=parts['key_alias'], creator_glob_id=parts['customer']))
self.set_local_path(localPath)
if _Debug:
lg.out(_DebugLevel, 'new Task created: %r' % self)
events.send('backup-task-created', data=dict(
number=self.number,
created=self.created,
backup_id=self.backupID,
key_id=self.keyID,
path_id=self.pathID,
customer_id=self.customerGlobID,
path=self.remotePath,
local_path=self.localPath,
remote_path=self.fullGlobPath,
))
def destroy(self, message=None):
lg.out(4, 'backup_control.Task-%d.destroy %s -> %s' % (
self.number, self.localPath, self.backupID))
if self.result_defer and not self.result_defer.called:
self.result_defer.cancel()
self.result_defer = None
events.send('backup-task-finished', data=dict(
number=self.number,
created=self.created,
backup_id=self.backupID,
key_id=self.keyID,
path_id=self.pathID,
customer_id=self.customerGlobID,
path=self.remotePath,
local_path=self.localPath,
remote_path=self.fullGlobPath,
message=message,
))
def set_path_id(self, pathID):
parts = global_id.ParseGlobalID(pathID)
self.pathID = pathID # source path to backup
self.customerGlobID = parts['customer']
self.customerIDURL = parts['idurl']
self.remotePath = parts['path'] # here it must be in 0/1/2 form
if parts['key_alias']:
self.set_key_id(my_keys.make_key_id(alias=parts['key_alias'], creator_glob_id=self.customerGlobID))
return parts
def set_key_id(self, key_id):
self.keyID = key_id
self.keyAlias = packetid.KeyAlias(self.keyID)
self.fullGlobPath = global_id.MakeGlobalID(
customer=self.customerGlobID, key_alias=self.keyAlias, path=self.remotePath)
self.fullCustomerID = global_id.MakeGlobalID(
customer=self.customerGlobID, key_alias=self.keyAlias)
def set_local_path(self, localPath):
self.localPath = localPath
def __repr__(self):
"""
Return a string like:
"Task-5: 0/1/2/3 from /home/veselin/Documents/myfile.txt"
"""
return 'Task-%d(%s from %s)' % (self.number, self.pathID, self.localPath)
# def _on_job_done(self, backupID, result):
# reactor.callLater(0, OnJobDone, backupID, result)
# if self.result_defer is not None:
# self.result_defer.callback((backupID, result))
# self.result_defer = None
# def _on_job_failed(self, backupID, err=None):
# if self.result_defer is not None:
# self.result_defer.errback((backupID, err))
# self.result_defer = None
# return err
def run(self):
"""
Runs a new ``Job`` from that ``Task``.
"""
iter_and_path = backup_fs.WalkByID(self.remotePath, iterID=backup_fs.fsID(self.customerIDURL))
if iter_and_path is None:
lg.out(4, 'backup_control.Task.run ERROR %s not found in the index' % self.remotePath)
# self.defer.callback('error', self.pathID)
# self._on_job_failed(self.pathID)
err = 'remote path "%s" not found in the catalog' % self.remotePath
OnTaskFailed(self.pathID, err)
return err
itemInfo, sourcePath = iter_and_path
if isinstance(itemInfo, dict):
try:
itemInfo = itemInfo[backup_fs.INFO_KEY]
except:
lg.exc()
# self._on_job_failed(self.pathID)
err = 'catalog item related to "%s" is broken' % self.remotePath
OnTaskFailed(self.pathID, err)
return err
if not self.localPath:
self.localPath = sourcePath
lg.out('backup_control.Task.run local path was populated from catalog: %s' % self.localPath)
if self.localPath != sourcePath:
lg.warn('local path is differ from catalog: %s != %s' % (self.localPath, sourcePath))
if not bpio.pathExist(self.localPath):
lg.warn('path not exist: %s' % self.localPath)
# self._on_job_failed(self.pathID)
err = 'local path "%s" not exist' % self.localPath
OnTaskFailed(self.pathID, err)
return err
# if os.path.isfile(self.localPath) and self.localPath != sourcePath:
# tmpfile.make(name, extension, prefix)
dataID = misc.NewBackupID()
if itemInfo.has_version(dataID):
# ups - we already have same version
# let's add 1,2,3... to the end to make absolutely unique version ID
i = 1
while itemInfo.has_version(dataID + str(i)):
i += 1
dataID += str(i)
self.backupID = packetid.MakeBackupID(
customer=self.fullCustomerID,
path_id=self.remotePath,
version=dataID,
)
if self.backupID in jobs():
lg.warn('backup job %s already started' % self.backupID)
return 'backup job %s already started' % self.backupID
try:
backup_fs.MakeLocalDir(settings.getLocalBackupsDir(), self.backupID)
except:
lg.exc()
lg.out(4, 'backup_control.Task.run ERROR creating destination folder for %s' % self.pathID)
# self.defer.callback('error', self.pathID)
# self._on_job_failed(self.backupID)
err = 'failed creating destination folder for "%s"' % self.backupID
return OnTaskFailed(self.backupID, err)
compress_mode = 'bz2' # 'none' # 'gz'
arcname = os.path.basename(sourcePath)
if bpio.pathIsDir(self.localPath):
backupPipe = backup_tar.backuptardir(self.localPath, arcname=arcname, compress=compress_mode)
else:
backupPipe = backup_tar.backuptarfile(self.localPath, arcname=arcname, compress=compress_mode)
backupPipe.make_nonblocking()
job = backup.backup(
self.backupID,
backupPipe,
finishCallback=OnJobDone,
blockResultCallback=OnBackupBlockReport,
blockSize=settings.getBackupBlockSize(),
sourcePath=self.localPath,
keyID=self.keyID or itemInfo.key_id,
)
jobs()[self.backupID] = job
itemInfo.add_version(dataID)
if itemInfo.type == backup_fs.DIR:
dirsize.ask(self.localPath, OnFoundFolderSize, (self.pathID, dataID))
else:
sz = os.path.getsize(self.localPath)
jobs()[self.backupID].totalSize = sz
itemInfo.set_size(sz)
backup_fs.Calculate()
Save()
jobs()[self.backupID].automat('start')
reactor.callLater(0, FireTaskStartedCallbacks, self.pathID, dataID)
lg.out(4, 'backup_control.Task-%d.run [%s/%s], size=%d, %s' % (
self.number, self.pathID, dataID, itemInfo.size, self.localPath))
return None
def cancel(self):
"""we want no answer, just let the dialog disappear"""
self.__removeFromScene()
Deferred.cancel(self)
class AsyncHttpClient(object):
"""
Supports json request payload on both HTTP GET and POST
Args:
uri : e.g. http://localhost/api/data
body :
method : GET or POST
headers : (optional)
connectTimeout : (optional)
"""
def __init__(self, **kwargs):
self.headers = {}
# uri, method, body
for k,v in kwargs.items():
setattr(self, k, v)
# Set default method to GET
if not kwargs.has_key('method'):
self.method = 'GET'
# Set default connectTimeout to 3
if not kwargs.has_key('connectTimeout'):
self.connectTimeout = 3.0
# Set default headers
self.headers['User-Agent'] = ['AsyncHttpRequest']
self.headers['Accept-Encoding'] = ['gzip']
self.body = None
self.agent = Agent(reactor, connectTimeout=self.connectTimeout)
self.__deferredResponse = Deferred()
if kwargs.has_key('body'):
if self.headers.has_key('Content-Type'):
for ctype in self.headers['Content-Type']:
# json request
if ctype.startswith('application/json'):
self.body = JsonBodyProducer(kwargs['body'])
break
# generic request
if self.body == None:
self.body = BodyProducer(kwargs['body'])
# Make the http request
self.__d_agent = self.agent.request(
self.method, self.url,
Headers(self.headers), self.body)
def __readResponseCallback(self, response, userCb, *cbargs):
response.deliverBody(AsyncHttpResponseProtocol(self.__deferredResponse, response.headers))
self.__deferredResponse.addCallback(userCb, *([response]+list(cbargs)))
return self.__deferredResponse
def __readErrorCallback(self, error, userCb, *cbargs):
self.__deferredResponse.addErrback(userCb, *cbargs)
def addResponseCallback(self, callback, *cbargs):
self.__d_agent.addCallback(self.__readResponseCallback, callback, *cbargs)
return self.__deferredResponse
def addResponseErrback(self, callback, *cbargs):
self.__d_agent.addErrback(self.__readErrorCallback, callback, *cbargs)
def cancelRequest(self):
try:
self.__deferredResponse.cancel()
self.__d_agent.cancel()
except Exception,e:
logger.debug(str(e))
def cancel(self):
self._cancel_timer()
return Deferred.cancel(self)
def cancel(self):
self._cancel_timer()
return Deferred.cancel(self)
class GeminiClientProtocol(LineReceiver, TimeoutMixin):
def __init__(self, request, maxsize, warnsize, timeout):
self.request = request
self.maxsize = maxsize
self.warnsize = warnsize
self.timout = timeout
self.reached_warnsize = False
self.request_url = urldefrag(self.request.url).url
self.response_header = b''
self.response_body = BytesIO()
self.response_size = 0
# Ideally this timer would start exactly when we send out the TCP SYN,
# but I don't know how to hook into that event with twisted.
self.start_time = time.time()
self.finished = Deferred(self.cancel)
def cancel(self, _):
self.transport.abortConnection()
def connectionMade(self):
self.setTimeout(self.timout)
self.sendLine(self.request_url.encode("utf-8"))
def timeoutConnection(self):
logger.error(
f"Getting {self.request} took longer than {self.timout} seconds."
)
self.transport.abortConnection()
def lineReceived(self, line):
logger.debug(f"{self.request.url}: Line received")
self.response_header = line
self.setRawMode()
def rawDataReceived(self, data):
if not self.response_size:
logger.debug(f"{self.request.url}: Data received ({len(data)})")
self.response_body.write(data)
self.response_size += len(data)
if self.maxsize and self.response_size > self.maxsize:
logger.error(
f"Received ({self.response_size}) bytes larger than download "
f"max size ({self.maxsize}) in request {self.request}."
)
# Clear buffer earlier to avoid keeping data in memory for a long time.
self.response_body.truncate(0)
self.finished.cancel()
if self.warnsize and self.response_size > self.warnsize and not self.reached_warnsize:
self.reached_warnsize = True
logger.warning(
f"Received more bytes than download warn size "
f"({self.warnsize}) in request {self.request}."
)
def connectionLost(self, reason=connectionDone):
logger.debug(f"{self.request.url}: Connection lost ({reason.value})")
self.setTimeout(None)
if self.finished.called:
return
self.request.meta['download_latency'] = time.time() - self.start_time
# Many gemini servers kill the connection uncleanly, i.e. ConnectionLost
if reason.check(ConnectionDone, ConnectionLost):
try:
response = self.build_response()
except Exception:
self.finished.errback()
else:
self.finished.callback(response)
else:
self.finished.errback(reason)
def build_response(self):
"""
Convert the response data into a pseudo-HTTP response.
"""
return GeminiResponse(
self.request_url,
gemini_header=self.response_header,
body=self.response_body.getvalue(),
certificate=self.transport.getPeerCertificate(),
ip_address=self.transport.getPeer().host,
)
class TimeoutDeferredTests(SynchronousTestCase):
"""
Tests for the method method ``timeout_deferred``
"""
def setUp(self):
"""
Create a clock and a deferred to be cancelled
"""
self.clock = Clock()
self.deferred = Deferred()
def test_propagates_result_if_success_before_timeout(self):
"""
The deferred callbacks with the result if it succeeds before the
timeout (e.g. timing out the deferred does not obscure the callback
value).
"""
clock = Clock()
d = Deferred()
timeout_deferred(d, 10, clock)
d.callback("Result")
self.assertEqual(self.successResultOf(d), "Result")
# the timeout never happens - no errback occurs
clock.advance(15)
self.assertIsNone(self.successResultOf(d))
def test_propagates_failure_if_failed_before_timeout(self):
"""
The deferred errbacks with the failure if it fails before the
timeout (e.g. timing out the deferred does not obscure the errback
failure).
"""
clock = Clock()
d = Deferred()
timeout_deferred(d, 10, clock)
d.errback(DummyException("fail"))
self.failureResultOf(d, DummyException)
# the timeout never happens - no further errback occurs
clock.advance(15)
self.assertIsNone(self.successResultOf(d))
def test_times_out_if_past_timeout(self):
"""
The deferred errbacks with a TimedOutError if the timeout occurs
before it either callbacks or errbacks.
"""
clock = Clock()
d = Deferred()
timeout_deferred(d, 10, clock)
self.assertNoResult(d)
clock.advance(15)
self.failureResultOf(d, TimedOutError)
def test_preserves_cancellation_function_callback(self):
"""
If a cancellation function that callbacks is provided to the deferred
being cancelled, its effects will not be overriden with a TimedOutError.
"""
d = Deferred(lambda c: c.callback('I was cancelled!'))
timeout_deferred(d, 10, self.clock)
self.assertNoResult(d)
self.clock.advance(15)
self.assertEqual(self.successResultOf(d), 'I was cancelled!')
def test_preserves_cancellation_function_errback(self):
"""
If a cancellation function that errbacks (with a non-CancelledError) is
provided to the deferred being cancelled, this other error will not be
converted to a TimedOutError.
"""
d = Deferred(lambda c: c.errback(DummyException('what!')))
timeout_deferred(d, 10, self.clock)
self.assertNoResult(d)
self.clock.advance(15)
self.failureResultOf(d, DummyException)
def test_preserves_early_cancellation_error(self):
"""
If the Deferred is manually cancelled before the timeout, it is not
re-cancelled (no AlreadyCancelledError), and the CancelledError is not
obscured
"""
timeout_deferred(self.deferred, 10, self.clock)
self.assertNoResult(self.deferred)
self.deferred.cancel()
self.failureResultOf(self.deferred, CancelledError)
self.clock.advance(15)
# no AlreadyCancelledError raised? Good.
def test_deferred_description_passed_to_TimedOutError(self):
"""
If a deferred_description is passed, the TimedOutError will have that
string as part of it's string representation.
"""
timeout_deferred(self.deferred, 5.3, self.clock,
deferred_description="It'sa ME!")
self.clock.advance(6)
f = self.failureResultOf(self.deferred, TimedOutError)
self.assertIn("It'sa ME! timed out after 5.3 seconds", str(f))
def root(request):
_d = Deferred()
_d.addErrback(cancelled.append)
request.notifyFinish().addCallback(lambda _: _d.cancel())
return _d
def cancel(self):
"""we want no answer, just let the dialog disappear"""
if self.dlg:
self.dlg.hide()
self.dlg = None
Deferred.cancel(self)
class AsyncHttpClient(object):
"""
Supports json request payload on both HTTP GET and POST
Args:
uri : e.g. http://localhost/api/data
body :
method : GET or POST
headers : (optional)
connectTimeout : (optional)
"""
def __init__(self, **kwargs):
self.headers = {}
# uri, method, body
for k, v in kwargs.items():
setattr(self, k, v)
# Set default method to GET
if not kwargs.has_key('method'):
self.method = 'GET'
# Set default connectTimeout to 3
if not kwargs.has_key('connectTimeout'):
self.connectTimeout = 3.0
# Set default headers
self.headers['User-Agent'] = ['AsyncHttpRequest']
self.headers['Accept-Encoding'] = ['gzip']
self.body = None
self.agent = Agent(reactor, connectTimeout=self.connectTimeout)
self.__deferredResponse = Deferred()
if kwargs.has_key('body'):
if self.headers.has_key('Content-Type'):
for ctype in self.headers['Content-Type']:
# json request
if ctype.startswith('application/json'):
self.body = JsonBodyProducer(kwargs['body'])
break
# generic request
if self.body == None:
self.body = BodyProducer(kwargs['body'])
# Make the http request
self.__d_agent = self.agent.request(self.method, self.url,
Headers(self.headers), self.body)
def __readResponseCallback(self, response, userCb, *cbargs):
response.deliverBody(
AsyncHttpResponseProtocol(self.__deferredResponse,
response.headers))
self.__deferredResponse.addCallback(userCb,
*([response] + list(cbargs)))
return self.__deferredResponse
def __readErrorCallback(self, error, userCb, *cbargs):
self.__deferredResponse.addErrback(userCb, *cbargs)
def addResponseCallback(self, callback, *cbargs):
self.__d_agent.addCallback(self.__readResponseCallback, callback,
*cbargs)
return self.__deferredResponse
def addResponseErrback(self, callback, *cbargs):
self.__d_agent.addErrback(self.__readErrorCallback, callback, *cbargs)
def cancelRequest(self):
try:
self.__deferredResponse.cancel()
self.__d_agent.cancel()
except Exception, e:
logger.debug(str(e))
