def test_subsequentBeforeTriggerFiresPriorBeforeDeferred(self):
"""
If a trigger added to the C{'before'} phase of an event calls back a
L{Deferred} returned by an earlier trigger in the C{'before'} phase of
the same event, the remaining C{'before'} triggers for that event
should be run and any further L{Deferred}s waited on before proceeding
to the C{'during'} events.
"""
eventType = 'test'
events = []
firstDeferred = Deferred()
secondDeferred = Deferred()
def firstBeforeTrigger():
return firstDeferred
def secondBeforeTrigger():
firstDeferred.callback(None)
def thirdBeforeTrigger():
events.append('before')
return secondDeferred
def duringTrigger():
events.append('during')
self.addTrigger('before', eventType, firstBeforeTrigger)
self.addTrigger('before', eventType, secondBeforeTrigger)
self.addTrigger('before', eventType, thirdBeforeTrigger)
self.addTrigger('during', eventType, duringTrigger)
self.assertEqual(events, [])
reactor.fireSystemEvent(eventType)
self.assertEqual(events, ['before'])
secondDeferred.callback(None)
self.assertEqual(events, ['before', 'during'])
def test_log_while_connecting(self):
connect_d = Deferred()
real_connect = self.endpoint.connect.side_effect
def _delay_connect(factory):
d = real_connect(factory)
connect_d.addCallback(lambda _: d)
return connect_d
self.endpoint.connect.side_effect = _delay_connect
client = ScribeClient(self.endpoint)
d = client.log('category', 'message')
self.assertEqual(self.client_proto.client.Log.call_count, 0)
d2 = client.log('category', 'message')
self.assertEqual(self.client_proto.client.Log.call_count, 0)
connect_d.callback(None)
self.assertEqual(self.assertFired(d), ttypes.ResultCode.OK)
self.assertEqual(self.assertFired(d2), ttypes.ResultCode.OK)
self.assertEqual(self.client_proto.client.Log.call_count, 2)
def _call(self, msg, cb, *args):
""" Internally used method which should be used to call the service.
@param msg: Request message which should be sent.
@param cb: Callback which should be called to process
response. The response will be the first
argument and additional arguments passed to the
_call method will be passed to the callback.
@param *args: Additional arguments which will be passed to the
callback.
"""
if not self._status:
raise InterfaceDisabledError('A disabled interface should not be '
'called.')
if not callable(cb):
raise TypeError('Callback has to be callable.')
uid = uuid4().hex
deferred = Deferred()
deferred.addCallback(cb, *args)
self._responses[uid] = deferred
self._conn.sendMessage(self._iTag, self._clsName, msg, uid)
def send_command(self, command, expect='OK'):
self.log('Sending: %r' % (command,))
resp = Deferred()
resp.addCallback(self.debug)
self.deferreds.append((expect, resp))
self.sendLine(command)
return resp
def get_package(self, request, from_ip=None):
commands = self._parse_request(request)
d = Deferred()
d.callback(commands)
return d
def test_wb_connect_after_timeout(self):
"""
Test an odd error scenario. If the zookeeper client succeeds in
connecting after a timeout, the connection should be closed, as
the connect deferred has already fired.
"""
mock_client = self.mocker.patch(self.client)
mock_client.close()
def close_state():
# Ensure the client state variable is correct after the close call.
self.client.connected = False
self.mocker.call(close_state)
self.mocker.replay()
task = DelayedCall(1, lambda: 1, None, None, None, None)
task.called = True
d = Deferred()
d.errback(ConnectionTimeoutException())
self.client._cb_connected(
task, d, None, zookeeper.CONNECTED_STATE, "/")
self.failUnlessFailure(d, ConnectionTimeoutException)
return d
def test_processTransportInterface(self):
"""
L{IReactorProcess.spawnProcess} connects the protocol passed to it
to a transport which provides L{IProcessTransport}.
"""
ended = Deferred()
protocol = _ShutdownCallbackProcessProtocol(ended)
reactor = self.buildReactor()
transport = reactor.spawnProcess(
protocol, sys.executable, [sys.executable, "-c", ""],
usePTY=self.usePTY)
# The transport is available synchronously, so we can check it right
# away (unlike many transport-based tests). This is convenient even
# though it's probably not how the spawnProcess interface should really
# work.
# We're not using verifyObject here because part of
# IProcessTransport is a lie - there are no getHost or getPeer
# methods. See #1124.
self.assertTrue(IProcessTransport.providedBy(transport))
# Let the process run and exit so we don't leave a zombie around.
ended.addCallback(lambda ignored: reactor.stop())
self.runReactor(reactor)
def test(self):
crawler = mock.MagicMock()
crawler.settings = CrawlerSettings()
crawler.settings.overrides['USER_AGENT'] = 'CustomAgent'
self.assertRaises(NotConfigured, RobotsTxtMiddleware, crawler)
crawler.settings.overrides['ROBOTSTXT_OBEY'] = True
crawler.engine.download = mock.MagicMock()
ROBOTS = re.sub(r'^\s+(?m)', '', '''
User-Agent: *
Disallow: /admin/
Disallow: /static/
''')
response = Response('http://site.local/robots.txt', body=ROBOTS)
def return_response(request, spider):
deferred = Deferred()
reactor.callFromThread(deferred.callback, response)
return deferred
crawler.engine.download.side_effect = return_response
middleware = RobotsTxtMiddleware(crawler)
spider = None # not actually used
# There is a bit of neglect in robotstxt.py: robots.txt is fetched asynchronously,
# and it is actually fetched only *after* first process_request completes.
# So, first process_request will always succeed.
# We defer test() because otherwise robots.txt download mock will be called after assertRaises failure.
self.assertIsNone(middleware.process_request(Request('http://site.local'), spider)) # not affected by robots.txt
def test(r):
self.assertIsNone(middleware.process_request(Request('http://site.local/allowed'), spider))
self.assertRaises(IgnoreRequest, middleware.process_request, Request('http://site.local/admin/main'), spider)
self.assertRaises(IgnoreRequest, middleware.process_request, Request('http://site.local/static/'), spider)
deferred = Deferred()
deferred.addCallback(test)
reactor.callFromThread(deferred.callback, None)
return deferred
def becomeGridHost(self, *args):
if self.clientObject.getLocalUser().gridHost:
for uuid in self.clientObject.users:
if self.clientObject.users[uuid].gridHostActive:
#TODO: Allow moderators to take gridhost from others.
showModalDialog(
self.window,
Gtk.MessageType.ERROR,
'The grid is already being hosted.'
)
return
#TODO: Show error dialogs on failures
self.setStatus('Loading OpenSim distribution...')
distribution = Distribution(self.clientObject.projectRoot, self.clientObject.externalhost, parent=self.window)
d = Deferred()
d.addCallback(self.startRobust)
distribution.load(d)
#TODO: Don't hardcode this
else:
showModalDialog(
self.window,
Gtk.MessageType.ERROR,
'You do not have permission to become the grid host.'
)
def with_timeout(timeout, d, reactor=reactor):
"""Returns a `Deferred` that is in all respects equivalent to `d`, e.g. when `cancel()` is called on it `Deferred`,
the wrapped `Deferred` will also be cancelled; however, a `Timeout` will be fired after the `timeout` number of
seconds if `d` has not fired by that time.
When a `Timeout` is raised, `d` will be cancelled. It is up to the caller to worry about how `d` handles
cancellation, i.e. whether it has full/true support for cancelling, or does cancelling it just prevent its callbacks
from being fired but doesn't cancel the underlying operation.
"""
if timeout is None or not isinstance(d, Deferred):
return d
ret = Deferred(canceller=lambda _: (d.cancel(), timeout_d.cancel()))
timeout_d = sleep(timeout, reactor)
timeout_d.addCallback(lambda _: (d.cancel(), ret.errback(Failure(Timeout())) if not ret.called else None))
timeout_d.addErrback(lambda f: f.trap(CancelledError))
d.addCallback(lambda result: (timeout_d.cancel(), ret.callback(result)))
d.addErrback(lambda f: (if_(not f.check(CancelledError), lambda: (timeout_d.cancel(), ret.errback(f))),))
return ret
def connect(self, host=None, port=None, spec=None, user=None, password=None, vhost=None,
heartbeat=None, clientClass=None):
host = host or self.host
port = port or self.port
spec = spec or self.spec
user = user or self.user
password = password or self.password
vhost = vhost or self.vhost
heartbeat = heartbeat or self.heartbeat
clientClass = clientClass or self.clientClass
delegate = TwistedDelegate()
onConn = Deferred()
p = clientClass(delegate, vhost, txamqp.spec.load(spec), heartbeat=heartbeat)
f = protocol._InstanceFactory(reactor, p, onConn)
c = reactor.connectTCP(host, port, f)
def errb(thefailure):
thefailure.trap(error.ConnectionRefusedError)
print "failed to connect to host: %s, port: %s; These tests are designed to run against a running instance" \
" of the %s AMQP broker on the given host and port. failure: %r" % (host, port, self.broker, thefailure,)
thefailure.raiseException()
onConn.addErrback(errb)
self.connectors.append(c)
client = yield onConn
yield client.authenticate(user, password)
returnValue(client)
def _writeTest(self, write):
"""
Helper for testing L{IProcessTransport} write functionality. This
method spawns a child process and gives C{write} a chance to write some
bytes to it. It then verifies that the bytes were actually written to
it (by relying on the child process to echo them back).
@param write: A two-argument callable. This is invoked with a process
transport and some bytes to write to it.
"""
reactor = self.buildReactor()
ended = Deferred()
protocol = _ShutdownCallbackProcessProtocol(ended)
bytes = "hello, world" + os.linesep
program = (
"import sys\n"
"sys.stdout.write(sys.stdin.readline())\n"
)
def startup():
transport = reactor.spawnProcess(
protocol, sys.executable, [sys.executable, "-c", program])
try:
write(transport, bytes)
except:
err(None, "Unhandled exception while writing")
transport.signalProcess('KILL')
reactor.callWhenRunning(startup)
ended.addCallback(lambda ignored: reactor.stop())
self.runReactor(reactor)
self.assertEqual(bytes, "".join(protocol.received[1]))
def test_rewind_stops_on_error(self):
"""
rewind errbacks it's completion deferred when it encounters an
error.
"""
called = [0]
def op(op_d):
called[0] += 1
return op_d
self.undo.push(op, None)
op_d1 = Deferred()
self.undo.push(op, op_d1)
d = self.undo.rewind()
self.assertNoResult(d)
class DummyOpFailure(Exception):
pass
op_d1.errback(DummyOpFailure())
self.assertEqual(called[0], 1)
self.failureResultOf(d, DummyOpFailure)
class TestBase(object):
def __init__(self, conn, iTag, testType):
self._conn = conn
self._iTag = iTag
self._testType = testType
self._data = []
def _activate(self):
pass
def _deactivate(self):
pass
def _run(self):
raise NotImplementedError
def run(self, _):
self._deferred = Deferred()
self._conn.reactor.callLater(1, self._activate)
self._conn.reactor.callLater(2, self._run)
return self._deferred
def _done(self):
self._deactivate()
self._deferred.callback(None)
def __str__(self):
return json.dumps({'type' : self._testType, 'data' : self._data})
def do_host_count(self, request, params = {}):
"""
Because process_host_count above is meant to handle its
own finalization, do_host_count always returns NOT_DONE_YET. Its
much simpler to make a tiny hack then hack process_host count to straddle
to different scenarios ( return string and process itself )
"""
#if this a polling request and it's not the first one
if 'ts' in params and params['ts'] != 0:
#if the caller is up to date or from the future
if params['ts'] > self.myStore.lastChange:
#hold the connection open
d = Deferred()
def deferred_host_count(self, request):
request.write(self.process_host_count)
request.finish()
#and notify them when something changes
d.addCallback(self.deferred_host_count, request)
self.myStore.onChange.observe(d)
return NOT_DONE_YET
#If no TS or TS is out of date, process NOW
return self.process_host_count(request)
def stop_daemon(self, host_id):
"""
Stops a running daemon.
:param host_id: the hash id of the host
:type host_id: string
"""
main_deferred = Deferred()
host = self.get_host(host_id)
if not host:
main_deferred.callback((False, _("Daemon doesn't exist")))
return main_deferred
try:
def on_connect(connected, c):
if not connected:
main_deferred.callback((False, _("Daemon not running")))
return
c.daemon.shutdown()
main_deferred.callback((True, ))
def on_connect_failed(reason):
main_deferred.callback((False, reason))
host, port, user, password = host[1:5]
c = Client()
d = c.connect(host, port, user, password)
d.addCallback(on_connect, c)
d.addErrback(on_connect_failed)
except:
main_deferred.callback((False, "An error occurred"))
return main_deferred
def test_disconnectedOnError(self):
"""
If the event handler raises an exception, the event is removed from the
reactor and the handler's C{connectionLost} method is called in the I/O
thread and the exception is logged.
"""
reactorThreadID = getThreadID()
reactor = self.buildReactor()
event = win32event.CreateEvent(None, False, False, None)
result = []
finished = Deferred()
finished.addBoth(result.append)
finished.addBoth(lambda ignored: reactor.stop())
listener = Listener(finished)
reactor.addEvent(event, listener, 'brokenOccurred')
reactor.callWhenRunning(win32event.SetEvent, event)
self.runReactor(reactor)
self.assertIsInstance(result[0], Failure)
result[0].trap(RuntimeError)
self.assertEqual(reactorThreadID, listener.connLostThreadID)
self.assertEqual(1, len(self.flushLoggedErrors(RuntimeError)))
def scale_down(self):
global overlay
n_workers = len(overlay.aws.workers)
d = Deferred()
for i in range(1, 1 + n_workers / 2):
d.addCallback(overlay.aws.term_worker(overlay.aws.workers[i]))
d.callback(0);
def test_double_subscribe(self):
handler = ApplicationSession()
MockTransport(handler)
event0 = Deferred()
event1 = Deferred()
subscription0 = yield handler.subscribe(
lambda: event0.callback(42), u'com.myapp.topic1')
subscription1 = yield handler.subscribe(
lambda: event1.callback('foo'), u'com.myapp.topic1')
# same topic, same ID
self.assertTrue(subscription0.id == subscription1.id)
# do a publish (MockTransport fakes the acknowledgement
# message) and then do an actual publish event. The IDs
# are the same, so we just do one Event.
publish = yield handler.publish(
u'com.myapp.topic1',
options=types.PublishOptions(acknowledge=True, exclude_me=False),
)
handler.onMessage(message.Event(subscription0.id, publish.id))
# ensure we actually got both callbacks
self.assertTrue(event0.called, "Missing callback")
self.assertTrue(event1.called, "Missing callback")
class HTTP11ClientProtocol(_newclient.HTTP11ClientProtocol):
def request(self, request):
if self._state != 'QUIESCENT':
return fail(RequestNotSent())
self._state = 'TRANSMITTING'
_requestDeferred = maybeDeferred(request.writeTo, self.transport)
self._finishedRequest = Deferred()
self._currentRequest = request
self._transportProxy = TransportProxyProducer(self.transport)
self._parser = HTTPClientParser(request, self._finishResponse)
self._parser.makeConnection(self._transportProxy)
self._responseDeferred = self._parser._responseDeferred
def cbRequestWrotten(ignored):
if self._state == 'TRANSMITTING':
self._state = 'WAITING'
self._responseDeferred.chainDeferred(self._finishedRequest)
def ebRequestWriting(err):
if self._state == 'TRANSMITTING':
self._state = 'GENERATION_FAILED'
self.transport.loseConnection()
self._finishedRequest.errback(
Failure(RequestGenerationFailed([err])))
else:
log.err(err, 'Error writing request, but not in valid state '
'to finalize request: %s' % self._state)
_requestDeferred.addCallbacks(cbRequestWrotten, ebRequestWriting)
return self._finishedRequest
def test_deferPreCommit(self):
"""
If callables passed to L{IAsyncTransaction.preCommit} return
L{Deferred}s, they will defer the actual commit operation until it has
fired.
"""
txn = self.createTransaction()
d = Deferred()
def wait():
wait.started = True
def executed(it):
wait.sqlResult = it
# To make sure the _underlying_ commit operation was Deferred, we
# have to execute some SQL to make sure it happens.
return (d.addCallback(lambda ignored: txn.execSQL("some test sql"))
.addCallback(executed))
wait.started = False
wait.sqlResult = None
txn.preCommit(wait)
result = self.resultOf(txn.commit())
self.flushHolders()
self.assertEquals(wait.started, True)
self.assertEquals(wait.sqlResult, None)
self.assertEquals(result, [])
d.callback(None)
# allow network I/O for pooled / networked implementation; there should
# be the commit message now.
self.flushHolders()
self.assertEquals(len(result), 1)
self.assertEquals(wait.sqlResult, [[1, "some test sql"]])
class DummyClient(DatagramProtocol):
def __init__(self, *args, **kwargs):
self.ready = Deferred()
def startProtocol(self):
self.ready.callback(None)
def execute_config(self, log, transaction_id, scaling_group, launch_config):
"""
see :meth:`ISupervisor.execute_config`
"""
job_id = generate_job_id(scaling_group.uuid)
completion_d = Deferred()
log = log.bind(job_id=job_id,
worker=launch_config['type'],
tenant_id=scaling_group.tenant_id)
assert launch_config['type'] == 'launch_server'
undo = InMemoryUndoStack(self.coiterate)
def when_fails(result):
log.msg("Encountered an error, rewinding {worker!r} job undo stack.",
exc=result.value)
ud = undo.rewind()
ud.addCallback(lambda _: result)
return ud
completion_d.addErrback(when_fails)
log.msg("Authenticating for tenant")
d = self.auth_function(scaling_group.tenant_id, log=log)
def when_authenticated((auth_token, service_catalog)):
log.msg("Executing launch config.")
return launch_server_v1.launch_server(
log,
self.region,
scaling_group,
service_catalog,
auth_token,
launch_config['args'], undo)
d.addCallback(when_authenticated)
def when_launch_server_completed(result):
# XXX: Something should be done with this data. Currently only enough
# to pass to the controller to store in the active state is returned
server_details, lb_info = result
log.msg("Done executing launch config.",
server_id=server_details['server']['id'])
return {
'id': server_details['server']['id'],
'links': server_details['server']['links'],
'name': server_details['server']['name'],
'lb_info': lb_info
}
d.addCallback(when_launch_server_completed)
self.deferred_pool.add(d)
d.chainDeferred(completion_d)
return succeed((job_id, completion_d))
def testTriggerSystemEvent2(self):
# one of the "before" trigger functions returns a deferred. A later
# "before" trigger fires the deferred. A third before runs. Then a
# "during" should be run. One of the failure modes for the old
# cReactor code is to start the "during" as soon as the deferred
# fires, rather than waiting for the "before" phase to be finished
l = []
d = Deferred()
d2 = Deferred()
def _returnDeferred(d=d):
return d
def _fireDeferred(d=d):
d.callback(None)
def _returnDeferred2(d2=d2):
return d2
def _appendToList(l=l):
l.append(1)
r = reactor
# to test this properly, the triggers must fire in this sequence:
# _returnDeferred, _fireDeferred, _returnDeferred2 . cReactor happens
# to run triggers in the order in which they were added.
self.addTrigger("before", "defer2", _returnDeferred)
self.addTrigger("before", "defer2", _fireDeferred)
self.addTrigger("before", "defer2", _returnDeferred2)
self.addTrigger("during", "defer2", _appendToList)
self.addTrigger("after", "defer2", _appendToList)
r.fireSystemEvent("defer2")
self.assertEquals(len(l), 0, "Event should not have fired yet.")
d2.callback(None)
self.assertEquals(len(l), 2)
def test_full_run(self):
"""Verify a functional agent start via the 'run' method.
This test requires Zookeeper running on the default port of localhost.
The mocked portions are to prevent the daemon start from altering the
test environment (sys.stdout/sys.stderr, and reactor start).
"""
zookeeper.set_debug_level(0)
started = Deferred()
class DummyAgent(BaseAgent):
started = False
def start(self):
started.callback(self)
def validate_started(agent):
self.assertTrue(agent.client.connected)
started.addCallback(validate_started)
pid_file = self.makeFile()
self.change_args("es-agent", "--zookeeper-servers", get_test_zookeeper_address(), "--pidfile", pid_file)
runner = self.mocker.patch(AgentRunner)
logger = self.mocker.patch(AppLogger)
logger.start(MATCH_APP)
runner.startReactor(None, sys.stdout, sys.stderr)
logger.stop()
self.mocker.replay()
DummyAgent.run()
return started
def testTriggerSystemEvent3(self):
# make sure reactor can survive the loss of an event type while
# waiting for a before-trigger's Deferred to fire
l = []
d = Deferred()
d2 = Deferred()
def _returnDeferred(d=d):
return d
def _appendToList(l=l):
l.append(1)
def _ignore(failure):
return None
r = reactor
b1 = self.addTrigger("before", "defer3", _returnDeferred)
b2 = self.addTrigger("after", "defer3", _appendToList)
r.fireSystemEvent("defer3")
self.assertEquals(len(l), 0, "Event should not have fired yet.")
self.removeTrigger(b1)
self.removeTrigger(b2)
try:
d.callback(None) # cReactor gives errback to deferred
except ValueError:
pass
self.assertEquals(len(l), 0)
d.addErrback(_ignore)
def testChildResolve(self):
# I've seen problems with reactor.run under gtk2reactor. Spawn a
# child which just does reactor.resolve after the reactor has
# started, fail if it does not complete in a timely fashion.
helperPath = os.path.abspath(self.mktemp())
helperFile = open(helperPath, 'w')
# Eeueuuggg
reactorName = reactor.__module__
helperFile.write(resolve_helper % {'reactor': reactorName})
helperFile.close()
env = os.environ.copy()
env['PYTHONPATH'] = os.pathsep.join(sys.path)
helperDeferred = Deferred()
helperProto = ChildResolveProtocol(helperDeferred)
reactor.spawnProcess(helperProto, sys.executable, ("python", "-u", helperPath), env)
def cbFinished((reason, output, error)):
# If the output is "done 127.0.0.1\n" we don't really care what
# else happened.
output = ''.join(output)
if output != 'done 127.0.0.1\n':
self.fail((
"The child process failed to produce the desired results:\n"
" Reason for termination was: %r\n"
" Output stream was: %r\n"
" Error stream was: %r\n") % (reason.getErrorMessage(), output, ''.join(error)))
helperDeferred.addCallback(cbFinished)
return helperDeferred
class TorIRC(IRCClient):
nickname = 'txsocksx-tor-irc'
nickservPassword = ''
def connectionMade(self):
self.sendLine('CAP REQ :sasl')
self.deferred = Deferred()
IRCClient.connectionMade(self)
def irc_CAP(self, prefix, params):
if params[1] != 'ACK' or params[2].split() != ['sasl']:
print 'sasl not available'
self.quit('')
sasl = ('{0}\0{0}\0{1}'.format(self.nickname, self.nickservPassword)).encode('base64').strip()
self.sendLine('AUTHENTICATE PLAIN')
self.sendLine('AUTHENTICATE ' + sasl)
def irc_903(self, prefix, params):
self.sendLine('CAP END')
def irc_904(self, prefix, params):
print 'sasl auth failed', params
self.quit('')
irc_905 = irc_904
def connectionLost(self, reason):
self.deferred.errback(reason)
def signedOn(self):
print 'signed on successfully'
self.quit('')
def get(self, tid):
""" Used to get a token by the BaseHandler, and whenever a
handler needs to token (usually because it wants to access
the store object).
tid -- The ID of the Token to get, it must have already been
created, usually by the get_token call to the AuthHandler.
"""
return_d = Deferred()
if self.tokens.get(tid) is not None:
# already in cache, return existing
return_d.callback(self.tokens.get(tid))
return return_d
# otherwise check the db
def token_cb(token_tuple):
if token_tuple is None:
return_d.callback(None)
return
username, password, boxid, appid, origin, clientip, server_id = token_tuple
token = Token(self.db, username, password, boxid, appid, origin, clientip, server_id)
self.add(token)
return_d.callback(token)
return
self.db.get_token(tid).addCallbacks(token_cb, return_d.errback)
return return_d
def run(self, regionEnd):
regionStart = 9000
log.msg("[NAT] Check Start")
d = Deferred()
d.addCallback(self.allEstablished)
self.ports2 = None
if regionStart != regionEnd and regionEnd >= 9000:
self.ports2 = [8002, 8003, 8004, regionStart, regionEnd]
else:
self.ports2 = [8002, 8003, 8004, regionStart]
self.ports = []
self.processports = []
for port in self.ports2:
hasProcessRunning = False
for name in self.clientObject.processes:
process = self.clientObject.processes[name]
if process.consolePort == port + 10000:
hasProcessRunning = True
if process.consolePort == 18000:
if port == 8002 or port == 8003 or port == 8004:
hasProcessRunning = True
if not hasProcessRunning:
self.ports += [port]
else:
if port == 8002:
self.processports += [18000]
elif port != 8003 and port != 8004: # We only want ROBUST once.
self.processports += [port + 10000]
self.count = len(self.ports)
self.service.start(d, self.count, self.ports)
def connect(factory, **kw):
return Deferred()
def trigger():
events.append("trigger")
d = Deferred()
d.addCallback(callback)
reactor.callLater(0, d.callback, None)
return d
def test_fire_when_all_fired(self):
"""
The ``Deferred`` returned by ``gather_deferreds`` does not fire until
all the supplied ``deferreds`` have either erred back or called back.
"""
d1 = Deferred()
d2 = Deferred()
d3 = Deferred()
gathering = gather_deferreds([d1, d2, d3])
# The second deferred fires first, with an error
d2.errback(ZeroDivisionError('test_consume_errors1'))
# But the gathered list does not fire...
self.assertNoResult(gathering)
# The remaining deferreds then callback...
d1.callback(None)
d3.callback(None)
# ...and the gathered list has now fired.
self.failureResultOf(gathering)
self.flushLoggedErrors(ZeroDivisionError)
def process_connect(self, packet):
"""
Process the initial Connect message from the MQTT client.
This should return a pair `(accept_conn, session_present)`, where
`accept_conn` is a return code:
0: connection accepted
1-5: connection refused (see MQTT spec 3.2.2.3)
"""
# Connect(client_id='paho/4E23D8C09DD9C6CF2C',
# flags=ConnectFlags(username=False,
# password=False,
# will=False,
# will_retain=False,
# will_qos=0,
# clean_session=True,
# reserved=False),
# keep_alive=60,
# will_topic=None,
# will_message=None,
# username=None,
# password=None)
self.log.info(
'WampMQTTServerProtocol.process_connect(packet={packet})',
packet=packet)
# we don't support session resumption: https://github.com/crossbario/crossbar/issues/892
if not packet.flags.clean_session:
self.log.warn(
'denying MQTT connect from {peer}, as the clients wants to resume a session (which we do not support)',
peer=peer2str(self.transport.getPeer()))
return succeed((1, False))
# we won't support QoS 2: https://github.com/crossbario/crossbar/issues/1046
if packet.flags.will and packet.flags.will_qos not in [0, 1]:
self.log.warn(
'denying MQTT connect from {peer}, as the clients wants to provide a "last will" event with QoS {will_qos} (and we only support QoS 0/1 here)',
peer=peer2str(self.transport.getPeer()),
will_qos=packet.flags.will_qos)
return succeed((1, False))
# this will be resolved when the MQTT connect handshake is completed
self._waiting_for_connect = Deferred()
roles = {
u"subscriber":
role.RoleSubscriberFeatures(payload_transparency=True,
pattern_based_subscription=True),
u"publisher":
role.RolePublisherFeatures(payload_transparency=True,
x_acknowledged_event_delivery=True)
}
realm = self.factory._options.get(u'realm', None)
authmethods = []
authextra = {
u'mqtt': {
u'client_id': packet.client_id,
u'will': bool(packet.flags.will),
u'will_topic': packet.will_topic
}
}
if ISSLTransport.providedBy(self.transport):
authmethods.append(u"tls")
if packet.username and packet.password:
authmethods.append(u"ticket")
msg = message.Hello(realm=realm,
roles=roles,
authmethods=authmethods,
authid=packet.username,
authextra=authextra)
self._pw_challenge = packet.password
else:
authmethods.append(u"anonymous")
msg = message.Hello(realm=realm,
roles=roles,
authmethods=authmethods,
authid=packet.client_id,
authextra=authextra)
self._wamp_session.onMessage(msg)
if packet.flags.will:
# it's unclear from the MQTT spec whether a) the publication of the last will
# is to happen in-band during "connect", and if it fails, deny the connection,
# or b) the last will publication happens _after_ "connect", and the connection
# succeeds regardless whether the last will publication succeeds or not.
#
# we opt for b) here!
#
@inlineCallbacks
@self._waiting_for_connect.addCallback
def process_will(res):
self.log.info()
payload_format, mapped_topic, options = yield self.factory.transform_mqtt(
packet.will_topic, packet.will_message)
request = util.id()
msg = message.Call(
request=request,
procedure=u"wamp.session.add_testament",
args=[
mapped_topic,
options.get('args', None),
options.get('kwargs', None),
{
# specifiy "retain" for when the testament (last will)
# will be auto-published by the broker later
u'retain': bool(packet.flags.will_retain)
}
])
self._wamp_session.onMessage(msg)
returnValue(res)
return self._waiting_for_connect
def __init__(self, request_cache, search_type, query):
super(ContentRequest, self).__init__(request_cache, u"request")
self.query = query
self.search_type = search_type
self.response = []
self.deferred = Deferred()
def test_disconnectAfterWriteAfterStartTLS(self):
"""
L{ITCPTransport.loseConnection} ends a connection which was set up with
L{ITLSTransport.startTLS} and which has recently been written to. This
is intended to verify that a socket send error masked by the TLS
implementation doesn't prevent the connection from being reported as
closed.
"""
class ShortProtocol(Protocol):
def connectionMade(self):
if not ITLSTransport.providedBy(self.transport):
# Functionality isn't available to be tested.
finished = self.factory.finished
self.factory.finished = None
finished.errback(SkipTest("No ITLSTransport support"))
return
# Switch the transport to TLS.
self.transport.startTLS(self.factory.context)
# Force TLS to really get negotiated. If nobody talks, nothing
# will happen.
self.transport.write(b"x")
def dataReceived(self, data):
# Stuff some bytes into the socket. This mostly has the effect
# of causing the next write to fail with ENOTCONN or EPIPE.
# With the pyOpenSSL implementation of ITLSTransport, the error
# is swallowed outside of the control of Twisted.
self.transport.write(b"y")
# Now close the connection, which requires a TLS close alert to
# be sent.
self.transport.loseConnection()
def connectionLost(self, reason):
# This is the success case. The client and the server want to
# get here.
finished = self.factory.finished
if finished is not None:
self.factory.finished = None
finished.callback(reason)
reactor = self.buildReactor()
serverFactory = ServerFactory()
serverFactory.finished = Deferred()
serverFactory.protocol = ShortProtocol
serverFactory.context = self.getServerContext()
clientFactory = ClientFactory()
clientFactory.finished = Deferred()
clientFactory.protocol = ShortProtocol
clientFactory.context = self.getClientContext()
clientFactory.context.method = serverFactory.context.method
lostConnectionResults = []
finished = DeferredList(
[serverFactory.finished, clientFactory.finished],
consumeErrors=True)
def cbFinished(results):
lostConnectionResults.extend([results[0][1], results[1][1]])
finished.addCallback(cbFinished)
port = reactor.listenTCP(0, serverFactory, interface="127.0.0.1")
self.addCleanup(port.stopListening)
connector = reactor.connectTCP(port.getHost().host,
port.getHost().port, clientFactory)
self.addCleanup(connector.disconnect)
finished.addCallback(lambda ign: reactor.stop())
self.runReactor(reactor)
lostConnectionResults[0].trap(ConnectionClosed)
lostConnectionResults[1].trap(ConnectionClosed)
def connect(factory, **kw):
d = Deferred()
reactor.callLater(
10, d.errback(RuntimeError("no connect for yo")))
return d
def test_requireContext(self):
"""
L{DeferredContext} raises a L{RuntimeError} if it is called without an
action context.
"""
self.assertRaises(RuntimeError, DeferredContext, Deferred())
class H2Connection(Protocol, TimeoutMixin):
"""
A class representing a single HTTP/2 connection.
This implementation of L{IProtocol} works hand in hand with L{H2Stream}.
This is because we have the requirement to register multiple producers for
a single HTTP/2 connection, one for each stream. The standard Twisted
interfaces don't really allow for this, so instead there's a custom
interface between the two objects that allows them to work hand-in-hand here.
@ivar conn: The HTTP/2 connection state machine.
@type conn: L{h2.connection.H2Connection}
@ivar streams: A mapping of stream IDs to L{H2Stream} objects, used to call
specific methods on streams when events occur.
@type streams: L{dict}, mapping L{int} stream IDs to L{H2Stream} objects.
@ivar priority: A HTTP/2 priority tree used to ensure that responses are
prioritised appropriately.
@type priority: L{priority.PriorityTree}
@ivar _consumerBlocked: A flag tracking whether or not the L{IConsumer}
that is consuming this data has asked us to stop producing.
@type _consumerBlocked: L{bool}
@ivar _sendingDeferred: A L{Deferred} used to restart the data-sending loop
when more response data has been produced. Will not be present if there
is outstanding data still to send.
@type _consumerBlocked: A L{twisted.internet.defer.Deferred}, or L{None}
@ivar _outboundStreamQueues: A map of stream IDs to queues, used to store
data blocks that are yet to be sent on the connection. These are used
both to handle producers that do not respect L{IConsumer} but also to
allow priority to multiplex data appropriately.
@type _outboundStreamQueues: A L{dict} mapping L{int} stream IDs to
L{collections.deque} queues, which contain either L{bytes} objects or
C{_END_STREAM_SENTINEL}.
@ivar _sender: A handle to the data-sending loop, allowing it to be
terminated if needed.
@type _sender: L{twisted.internet.task.LoopingCall}
@ivar abortTimeout: The number of seconds to wait after we attempt to shut
the transport down cleanly to give up and forcibly terminate it. This
is only used when we time a connection out, to prevent errors causing
the FD to get leaked. If this is L{None}, we will wait forever.
@type abortTimeout: L{int}
@ivar _abortingCall: The L{twisted.internet.base.DelayedCall} that will be
used to forcibly close the transport if it doesn't close cleanly.
@type _abortingCall: L{twisted.internet.base.DelayedCall}
"""
factory = None
site = None
abortTimeout = 15
_log = Logger()
_abortingCall = None
def __init__(self, reactor=None):
config = h2.config.H2Configuration(client_side=False, header_encoding=None)
self.conn = h2.connection.H2Connection(config=config)
self.streams = {}
self.priority = priority.PriorityTree()
self._consumerBlocked = None
self._sendingDeferred = None
self._outboundStreamQueues = {}
self._streamCleanupCallbacks = {}
self._stillProducing = True
# Limit the number of buffered control frame (e.g. PING and
# SETTINGS) bytes.
self._maxBufferedControlFrameBytes = 1024 * 17
self._bufferedControlFrames = deque()
self._bufferedControlFrameBytes = 0
if reactor is None:
from twisted.internet import reactor
self._reactor = reactor
# Start the data sending function.
self._reactor.callLater(0, self._sendPrioritisedData)
# Implementation of IProtocol
def connectionMade(self):
"""
Called by the reactor when a connection is received. May also be called
by the L{twisted.web.http._GenericHTTPChannelProtocol} during upgrade
to HTTP/2.
"""
self.setTimeout(self.timeOut)
self.conn.initiate_connection()
self.transport.write(self.conn.data_to_send())
def dataReceived(self, data):
"""
Called whenever a chunk of data is received from the transport.
@param data: The data received from the transport.
@type data: L{bytes}
"""
try:
events = self.conn.receive_data(data)
except h2.exceptions.ProtocolError:
stillActive = self._tryToWriteControlData()
if stillActive:
self.transport.loseConnection()
self.connectionLost(Failure(), _cancelTimeouts=False)
return
# Only reset the timeout if we've received an actual H2
# protocol message
self.resetTimeout()
for event in events:
if isinstance(event, h2.events.RequestReceived):
self._requestReceived(event)
elif isinstance(event, h2.events.DataReceived):
self._requestDataReceived(event)
elif isinstance(event, h2.events.StreamEnded):
self._requestEnded(event)
elif isinstance(event, h2.events.StreamReset):
self._requestAborted(event)
elif isinstance(event, h2.events.WindowUpdated):
self._handleWindowUpdate(event)
elif isinstance(event, h2.events.PriorityUpdated):
self._handlePriorityUpdate(event)
elif isinstance(event, h2.events.ConnectionTerminated):
self.transport.loseConnection()
self.connectionLost(
Failure(ConnectionLost("Remote peer sent GOAWAY")),
_cancelTimeouts=False,
)
self._tryToWriteControlData()
def timeoutConnection(self):
"""
Called when the connection has been inactive for
L{self.timeOut<twisted.protocols.policies.TimeoutMixin.timeOut>}
seconds. Cleanly tears the connection down, attempting to notify the
peer if needed.
We override this method to add two extra bits of functionality:
- We want to log the timeout.
- We want to send a GOAWAY frame indicating that the connection is
being terminated, and whether it was clean or not. We have to do this
before the connection is torn down.
"""
self._log.info("Timing out client {client}", client=self.transport.getPeer())
# Check whether there are open streams. If there are, we're going to
# want to use the error code PROTOCOL_ERROR. If there aren't, use
# NO_ERROR.
if self.conn.open_outbound_streams > 0 or self.conn.open_inbound_streams > 0:
error_code = h2.errors.ErrorCodes.PROTOCOL_ERROR
else:
error_code = h2.errors.ErrorCodes.NO_ERROR
self.conn.close_connection(error_code=error_code)
self.transport.write(self.conn.data_to_send())
# Don't let the client hold this connection open too long.
if self.abortTimeout is not None:
# We use self.callLater because that's what TimeoutMixin does, even
# though we have a perfectly good reactor sitting around. See
# https://twistedmatrix.com/trac/ticket/8488.
self._abortingCall = self.callLater(
self.abortTimeout, self.forceAbortClient
)
# We're done, throw the connection away.
self.transport.loseConnection()
def forceAbortClient(self):
"""
Called if C{abortTimeout} seconds have passed since the timeout fired,
and the connection still hasn't gone away. This can really only happen
on extremely bad connections or when clients are maliciously attempting
to keep connections open.
"""
self._log.info(
"Forcibly timing out client: {client}", client=self.transport.getPeer()
)
# We want to lose track of the _abortingCall so that no-one tries to
# cancel it.
self._abortingCall = None
self.transport.abortConnection()
def connectionLost(self, reason, _cancelTimeouts=True):
"""
Called when the transport connection is lost.
Informs all outstanding response handlers that the connection
has been lost, and cleans up all internal state.
@param reason: See L{IProtocol.connectionLost}
@param _cancelTimeouts: Propagate the C{reason} to this
connection's streams but don't cancel any timers, so that
peers who never read the data we've written are eventually
timed out.
"""
self._stillProducing = False
if _cancelTimeouts:
self.setTimeout(None)
for stream in self.streams.values():
stream.connectionLost(reason)
for streamID in list(self.streams.keys()):
self._requestDone(streamID)
# If we were going to force-close the transport, we don't have to now.
if _cancelTimeouts and self._abortingCall is not None:
self._abortingCall.cancel()
self._abortingCall = None
# Implementation of IPushProducer
#
# Here's how we handle IPushProducer. We have multiple outstanding
# H2Streams. Each of these exposes an IConsumer interface to the response
# handler that allows it to push data into the H2Stream. The H2Stream then
# writes the data into the H2Connection object.
#
# The H2Connection needs to manage these writes to account for:
#
# - flow control
# - priority
#
# We manage each of these in different ways.
#
# For flow control, we simply use the equivalent of the IPushProducer
# interface. We simply tell the H2Stream: "Hey, you can't send any data
# right now, sorry!". When that stream becomes unblocked, we free it up
# again. This allows the H2Stream to propagate this backpressure up the
# chain.
#
# For priority, we need to keep a backlog of data frames that we can send,
# and interleave them appropriately. This backlog is most sensibly kept in
# the H2Connection object itself. We keep one queue per stream, which is
# where the writes go, and then we have a loop that manages popping these
# streams off in priority order.
#
# Logically then, we go as follows:
#
# 1. Stream calls writeDataToStream(). This causes a DataFrame to be placed
# on the queue for that stream. It also informs the priority
# implementation that this stream is unblocked.
# 2. The _sendPrioritisedData() function spins in a tight loop. Each
# iteration it asks the priority implementation which stream should send
# next, and pops a data frame off that stream's queue. If, after sending
# that frame, there is no data left on that stream's queue, the function
# informs the priority implementation that the stream is blocked.
#
# If all streams are blocked, or if there are no outstanding streams, the
# _sendPrioritisedData function waits to be awoken when more data is ready
# to send.
#
# Note that all of this only applies to *data*. Headers and other control
# frames deliberately skip this processing as they are not subject to flow
# control or priority constraints. Instead, they are stored in their own buffer
# which is used primarily to detect excessive buffering.
def stopProducing(self):
"""
Stop producing data.
This tells the L{H2Connection} that its consumer has died, so it must
stop producing data for good.
"""
self.connectionLost(Failure(ConnectionLost("Producing stopped")))
def pauseProducing(self):
"""
Pause producing data.
Tells the L{H2Connection} that it has produced too much data to process
for the time being, and to stop until resumeProducing() is called.
"""
self._consumerBlocked = Deferred()
# Ensure pending control data (if any) are sent first.
self._consumerBlocked.addCallback(self._flushBufferedControlData)
def resumeProducing(self):
"""
Resume producing data.
This tells the L{H2Connection} to re-add itself to the main loop and
produce more data for the consumer.
"""
if self._consumerBlocked is not None:
d = self._consumerBlocked
self._consumerBlocked = None
d.callback(None)
def _sendPrioritisedData(self, *args):
"""
The data sending loop. This function repeatedly calls itself, either
from L{Deferred}s or from
L{reactor.callLater<twisted.internet.interfaces.IReactorTime.callLater>}
This function sends data on streams according to the rules of HTTP/2
priority. It ensures that the data from each stream is interleved
according to the priority signalled by the client, making sure that the
connection is used with maximal efficiency.
This function will execute if data is available: if all data is
exhausted, the function will place a deferred onto the L{H2Connection}
object and wait until it is called to resume executing.
"""
# If producing has stopped, we're done. Don't reschedule ourselves
if not self._stillProducing:
return
stream = None
while stream is None:
try:
stream = next(self.priority)
except priority.DeadlockError:
# All streams are currently blocked or not progressing. Wait
# until a new one becomes available.
assert self._sendingDeferred is None
self._sendingDeferred = Deferred()
self._sendingDeferred.addCallback(self._sendPrioritisedData)
return
# Wait behind the transport.
if self._consumerBlocked is not None:
self._consumerBlocked.addCallback(self._sendPrioritisedData)
return
self.resetTimeout()
remainingWindow = self.conn.local_flow_control_window(stream)
frameData = self._outboundStreamQueues[stream].popleft()
maxFrameSize = min(self.conn.max_outbound_frame_size, remainingWindow)
if frameData is _END_STREAM_SENTINEL:
# There's no error handling here even though this can throw
# ProtocolError because we really shouldn't encounter this problem.
# If we do, that's a nasty bug.
self.conn.end_stream(stream)
self.transport.write(self.conn.data_to_send())
# Clean up the stream
self._requestDone(stream)
else:
# Respect the max frame size.
if len(frameData) > maxFrameSize:
excessData = frameData[maxFrameSize:]
frameData = frameData[:maxFrameSize]
self._outboundStreamQueues[stream].appendleft(excessData)
# There's deliberately no error handling here, because this just
# absolutely should not happen.
# If for whatever reason the max frame length is zero and so we
# have no frame data to send, don't send any.
if frameData:
self.conn.send_data(stream, frameData)
self.transport.write(self.conn.data_to_send())
# If there's no data left, this stream is now blocked.
if not self._outboundStreamQueues[stream]:
self.priority.block(stream)
# Also, if the stream's flow control window is exhausted, tell it
# to stop.
if self.remainingOutboundWindow(stream) <= 0:
self.streams[stream].flowControlBlocked()
self._reactor.callLater(0, self._sendPrioritisedData)
# Internal functions.
def _requestReceived(self, event):
"""
Internal handler for when a request has been received.
@param event: The Hyper-h2 event that encodes information about the
received request.
@type event: L{h2.events.RequestReceived}
"""
stream = H2Stream(
event.stream_id,
self,
event.headers,
self.requestFactory,
self.site,
self.factory,
)
self.streams[event.stream_id] = stream
self._streamCleanupCallbacks[event.stream_id] = Deferred()
self._outboundStreamQueues[event.stream_id] = deque()
# Add the stream to the priority tree but immediately block it.
try:
self.priority.insert_stream(event.stream_id)
except priority.DuplicateStreamError:
# Stream already in the tree. This can happen if we received a
# PRIORITY frame before a HEADERS frame. Just move on: we set the
# stream up properly in _handlePriorityUpdate.
pass
else:
self.priority.block(event.stream_id)
def _requestDataReceived(self, event):
"""
Internal handler for when a chunk of data is received for a given
request.
@param event: The Hyper-h2 event that encodes information about the
received data.
@type event: L{h2.events.DataReceived}
"""
stream = self.streams[event.stream_id]
stream.receiveDataChunk(event.data, event.flow_controlled_length)
def _requestEnded(self, event):
"""
Internal handler for when a request is complete, and we expect no
further data for that request.
@param event: The Hyper-h2 event that encodes information about the
completed stream.
@type event: L{h2.events.StreamEnded}
"""
stream = self.streams[event.stream_id]
stream.requestComplete()
def _requestAborted(self, event):
"""
Internal handler for when a request is aborted by a remote peer.
@param event: The Hyper-h2 event that encodes information about the
reset stream.
@type event: L{h2.events.StreamReset}
"""
stream = self.streams[event.stream_id]
stream.connectionLost(
Failure(ConnectionLost("Stream reset with code %s" % event.error_code))
)
self._requestDone(event.stream_id)
def _handlePriorityUpdate(self, event):
"""
Internal handler for when a stream priority is updated.
@param event: The Hyper-h2 event that encodes information about the
stream reprioritization.
@type event: L{h2.events.PriorityUpdated}
"""
try:
self.priority.reprioritize(
stream_id=event.stream_id,
depends_on=event.depends_on or None,
weight=event.weight,
exclusive=event.exclusive,
)
except priority.MissingStreamError:
# A PRIORITY frame arrived before the HEADERS frame that would
# trigger us to insert the stream into the tree. That's fine: we
# can create the stream here and mark it as blocked.
self.priority.insert_stream(
stream_id=event.stream_id,
depends_on=event.depends_on or None,
weight=event.weight,
exclusive=event.exclusive,
)
self.priority.block(event.stream_id)
def writeHeaders(self, version, code, reason, headers, streamID):
"""
Called by L{twisted.web.http.Request} objects to write a complete set
of HTTP headers to a stream.
@param version: The HTTP version in use. Unused in HTTP/2.
@type version: L{bytes}
@param code: The HTTP status code to write.
@type code: L{bytes}
@param reason: The HTTP reason phrase to write. Unused in HTTP/2.
@type reason: L{bytes}
@param headers: The headers to write to the stream.
@type headers: L{twisted.web.http_headers.Headers}
@param streamID: The ID of the stream to write the headers to.
@type streamID: L{int}
"""
headers.insert(0, (b":status", code))
try:
self.conn.send_headers(streamID, headers)
except h2.exceptions.StreamClosedError:
# Stream was closed by the client at some point. We need to not
# explode here: just swallow the error. That's what write() does
# when a connection is lost, so that's what we do too.
return
else:
self._tryToWriteControlData()
def writeDataToStream(self, streamID, data):
"""
May be called by L{H2Stream} objects to write response data to a given
stream. Writes a single data frame.
@param streamID: The ID of the stream to write the data to.
@type streamID: L{int}
@param data: The data chunk to write to the stream.
@type data: L{bytes}
"""
self._outboundStreamQueues[streamID].append(data)
# There's obviously no point unblocking this stream and the sending
# loop if the data can't actually be sent, so confirm that there's
# some room to send data.
if self.conn.local_flow_control_window(streamID) > 0:
self.priority.unblock(streamID)
if self._sendingDeferred is not None:
d = self._sendingDeferred
self._sendingDeferred = None
d.callback(streamID)
if self.remainingOutboundWindow(streamID) <= 0:
self.streams[streamID].flowControlBlocked()
def endRequest(self, streamID):
"""
Called by L{H2Stream} objects to signal completion of a response.
@param streamID: The ID of the stream to write the data to.
@type streamID: L{int}
"""
self._outboundStreamQueues[streamID].append(_END_STREAM_SENTINEL)
self.priority.unblock(streamID)
if self._sendingDeferred is not None:
d = self._sendingDeferred
self._sendingDeferred = None
d.callback(streamID)
def abortRequest(self, streamID):
"""
Called by L{H2Stream} objects to request early termination of a stream.
This emits a RstStream frame and then removes all stream state.
@param streamID: The ID of the stream to write the data to.
@type streamID: L{int}
"""
self.conn.reset_stream(streamID)
stillActive = self._tryToWriteControlData()
if stillActive:
self._requestDone(streamID)
def _requestDone(self, streamID):
"""
Called internally by the data sending loop to clean up state that was
being used for the stream. Called when the stream is complete.
@param streamID: The ID of the stream to clean up state for.
@type streamID: L{int}
"""
del self._outboundStreamQueues[streamID]
self.priority.remove_stream(streamID)
del self.streams[streamID]
cleanupCallback = self._streamCleanupCallbacks.pop(streamID)
cleanupCallback.callback(streamID)
def remainingOutboundWindow(self, streamID):
"""
Called to determine how much room is left in the send window for a
given stream. Allows us to handle blocking and unblocking producers.
@param streamID: The ID of the stream whose flow control window we'll
check.
@type streamID: L{int}
@return: The amount of room remaining in the send window for the given
stream, including the data queued to be sent.
@rtype: L{int}
"""
# TODO: This involves a fair bit of looping and computation for
# something that is called a lot. Consider caching values somewhere.
windowSize = self.conn.local_flow_control_window(streamID)
sendQueue = self._outboundStreamQueues[streamID]
alreadyConsumed = sum(
len(chunk) for chunk in sendQueue if chunk is not _END_STREAM_SENTINEL
)
return windowSize - alreadyConsumed
def _handleWindowUpdate(self, event):
"""
Manage flow control windows.
Streams that are blocked on flow control will register themselves with
the connection. This will fire deferreds that wake those streams up and
allow them to continue processing.
@param event: The Hyper-h2 event that encodes information about the
flow control window change.
@type event: L{h2.events.WindowUpdated}
"""
streamID = event.stream_id
if streamID:
if not self._streamIsActive(streamID):
# We may have already cleaned up our stream state, making this
# a late WINDOW_UPDATE frame. That's fine: the update is
# unnecessary but benign. We'll ignore it.
return
# If we haven't got any data to send, don't unblock the stream. If
# we do, we'll eventually get an exception inside the
# _sendPrioritisedData loop some time later.
if self._outboundStreamQueues.get(streamID):
self.priority.unblock(streamID)
self.streams[streamID].windowUpdated()
else:
# Update strictly applies to all streams.
for stream in self.streams.values():
stream.windowUpdated()
# If we still have data to send for this stream, unblock it.
if self._outboundStreamQueues.get(stream.streamID):
self.priority.unblock(stream.streamID)
def getPeer(self):
"""
Get the remote address of this connection.
Treat this method with caution. It is the unfortunate result of the
CGI and Jabber standards, but should not be considered reliable for
the usual host of reasons; port forwarding, proxying, firewalls, IP
masquerading, etc.
@return: An L{IAddress} provider.
"""
return self.transport.getPeer()
def getHost(self):
"""
Similar to getPeer, but returns an address describing this side of the
connection.
@return: An L{IAddress} provider.
"""
return self.transport.getHost()
def openStreamWindow(self, streamID, increment):
"""
Open the stream window by a given increment.
@param streamID: The ID of the stream whose window needs to be opened.
@type streamID: L{int}
@param increment: The amount by which the stream window must be
incremented.
@type increment: L{int}
"""
self.conn.acknowledge_received_data(increment, streamID)
self._tryToWriteControlData()
def _isSecure(self):
"""
Returns L{True} if this channel is using a secure transport.
@returns: L{True} if this channel is secure.
@rtype: L{bool}
"""
# A channel is secure if its transport is ISSLTransport.
return ISSLTransport(self.transport, None) is not None
def _send100Continue(self, streamID):
"""
Sends a 100 Continue response, used to signal to clients that further
processing will be performed.
@param streamID: The ID of the stream that needs the 100 Continue
response
@type streamID: L{int}
"""
headers = [(b":status", b"100")]
self.conn.send_headers(headers=headers, stream_id=streamID)
self._tryToWriteControlData()
def _respondToBadRequestAndDisconnect(self, streamID):
"""
This is a quick and dirty way of responding to bad requests.
As described by HTTP standard we should be patient and accept the
whole request from the client before sending a polite bad request
response, even in the case when clients send tons of data.
Unlike in the HTTP/1.1 case, this does not actually disconnect the
underlying transport: there's no need. This instead just sends a 400
response and terminates the stream.
@param streamID: The ID of the stream that needs the 100 Continue
response
@type streamID: L{int}
"""
headers = [(b":status", b"400")]
self.conn.send_headers(headers=headers, stream_id=streamID, end_stream=True)
stillActive = self._tryToWriteControlData()
if stillActive:
stream = self.streams[streamID]
stream.connectionLost(Failure(ConnectionLost("Invalid request")))
self._requestDone(streamID)
def _streamIsActive(self, streamID):
"""
Checks whether Twisted has still got state for a given stream and so
can process events for that stream.
@param streamID: The ID of the stream that needs processing.
@type streamID: L{int}
@return: Whether the stream still has state allocated.
@rtype: L{bool}
"""
return streamID in self.streams
def _tryToWriteControlData(self):
"""
Checks whether the connection is blocked on flow control and,
if it isn't, writes any buffered control data.
@return: L{True} if the connection is still active and
L{False} if it was aborted because too many bytes have
been written but not consumed by the other end.
"""
bufferedBytes = self.conn.data_to_send()
if not bufferedBytes:
return True
if self._consumerBlocked is None and not self._bufferedControlFrames:
# The consumer isn't blocked, and we don't have any buffered frames:
# write this directly.
self.transport.write(bufferedBytes)
return True
else:
# Either the consumer is blocked or we have buffered frames. If the
# consumer is blocked, we'll write this when we unblock. If we have
# buffered frames, we have presumably been re-entered from
# transport.write, and so to avoid reordering issues we'll buffer anyway.
self._bufferedControlFrames.append(bufferedBytes)
self._bufferedControlFrameBytes += len(bufferedBytes)
if self._bufferedControlFrameBytes >= self._maxBufferedControlFrameBytes:
maxBuffCtrlFrameBytes = self._maxBufferedControlFrameBytes
self._log.error(
"Maximum number of control frame bytes buffered: "
"{bufferedControlFrameBytes} > = "
"{maxBufferedControlFrameBytes}. "
"Aborting connection to client: {client} ",
bufferedControlFrameBytes=self._bufferedControlFrameBytes,
maxBufferedControlFrameBytes=maxBuffCtrlFrameBytes,
client=self.transport.getPeer(),
)
# We've exceeded a reasonable buffer size for max buffered
# control frames. This is a denial of service risk, so we're
# going to drop this connection.
self.transport.abortConnection()
self.connectionLost(Failure(ExcessiveBufferingError()))
return False
return True
def _flushBufferedControlData(self, *args):
"""
Called when the connection is marked writable again after being marked unwritable.
Attempts to flush buffered control data if there is any.
"""
# To respect backpressure here we send each write in order, paying attention to whether
# we got blocked
while self._consumerBlocked is None and self._bufferedControlFrames:
nextWrite = self._bufferedControlFrames.popleft()
self._bufferedControlFrameBytes -= len(nextWrite)
self.transport.write(nextWrite)
def main(reactor, port="stdio:"):
endpoint = serverFromString(reactor, port)
flowFount = yield flowFountFromEndpoint(endpoint)
flowFount.flowTo(Listener(Hub().newParticipantFlow))
yield Deferred()
def deferredGenerator():
return Deferred().addCallback(IPBusPacket).addCallback(j).addCallback(
buildResponsePacket) #.addCallback(h.record)
def test_addActionFinishResult(self):
"""
L{DeferredContext.addActionFinish} returns the L{Deferred}.
"""
d = Deferred()
self.assertIs(d, DeferredContext(d).addActionFinish())
def timeout_deferred(
deferred: defer.Deferred,
timeout: float,
reactor: IReactorTime,
) -> defer.Deferred:
"""The in built twisted `Deferred.addTimeout` fails to time out deferreds
that have a canceller that throws exceptions. This method creates a new
deferred that wraps and times out the given deferred, correctly handling
the case where the given deferred's canceller throws.
(See https://twistedmatrix.com/trac/ticket/9534)
NOTE: Unlike `Deferred.addTimeout`, this function returns a new deferred.
NOTE: the TimeoutError raised by the resultant deferred is
twisted.internet.defer.TimeoutError, which is *different* to the built-in
TimeoutError, as well as various other TimeoutErrors you might have imported.
Args:
deferred: The Deferred to potentially timeout.
timeout: Timeout in seconds
reactor: The twisted reactor to use
Returns:
A new Deferred, which will errback with defer.TimeoutError on timeout.
"""
new_d = defer.Deferred()
timed_out = [False]
def time_it_out():
timed_out[0] = True
try:
deferred.cancel()
except: # noqa: E722, if we throw any exception it'll break time outs
logger.exception("Canceller failed during timeout")
# the cancel() call should have set off a chain of errbacks which
# will have errbacked new_d, but in case it hasn't, errback it now.
if not new_d.called:
new_d.errback(
defer.TimeoutError("Timed out after %gs" % (timeout, )))
delayed_call = reactor.callLater(timeout, time_it_out)
def convert_cancelled(value: failure.Failure):
# if the original deferred was cancelled, and our timeout has fired, then
# the reason it was cancelled was due to our timeout. Turn the CancelledError
# into a TimeoutError.
if timed_out[0] and value.check(CancelledError):
raise defer.TimeoutError("Timed out after %gs" % (timeout, ))
return value
deferred.addErrback(convert_cancelled)
def cancel_timeout(result):
# stop the pending call to cancel the deferred if it's been fired
if delayed_call.active():
delayed_call.cancel()
return result
deferred.addBoth(cancel_timeout)
def success_cb(val):
if not new_d.called:
new_d.callback(val)
def failure_cb(val):
if not new_d.called:
new_d.errback(val)
deferred.addCallbacks(success_cb, failure_cb)
return new_d
def _sendPrioritisedData(self, *args):
"""
The data sending loop. This function repeatedly calls itself, either
from L{Deferred}s or from
L{reactor.callLater<twisted.internet.interfaces.IReactorTime.callLater>}
This function sends data on streams according to the rules of HTTP/2
priority. It ensures that the data from each stream is interleved
according to the priority signalled by the client, making sure that the
connection is used with maximal efficiency.
This function will execute if data is available: if all data is
exhausted, the function will place a deferred onto the L{H2Connection}
object and wait until it is called to resume executing.
"""
# If producing has stopped, we're done. Don't reschedule ourselves
if not self._stillProducing:
return
stream = None
while stream is None:
try:
stream = next(self.priority)
except priority.DeadlockError:
# All streams are currently blocked or not progressing. Wait
# until a new one becomes available.
assert self._sendingDeferred is None
self._sendingDeferred = Deferred()
self._sendingDeferred.addCallback(self._sendPrioritisedData)
return
# Wait behind the transport.
if self._consumerBlocked is not None:
self._consumerBlocked.addCallback(self._sendPrioritisedData)
return
self.resetTimeout()
remainingWindow = self.conn.local_flow_control_window(stream)
frameData = self._outboundStreamQueues[stream].popleft()
maxFrameSize = min(self.conn.max_outbound_frame_size, remainingWindow)
if frameData is _END_STREAM_SENTINEL:
# There's no error handling here even though this can throw
# ProtocolError because we really shouldn't encounter this problem.
# If we do, that's a nasty bug.
self.conn.end_stream(stream)
self.transport.write(self.conn.data_to_send())
# Clean up the stream
self._requestDone(stream)
else:
# Respect the max frame size.
if len(frameData) > maxFrameSize:
excessData = frameData[maxFrameSize:]
frameData = frameData[:maxFrameSize]
self._outboundStreamQueues[stream].appendleft(excessData)
# There's deliberately no error handling here, because this just
# absolutely should not happen.
# If for whatever reason the max frame length is zero and so we
# have no frame data to send, don't send any.
if frameData:
self.conn.send_data(stream, frameData)
self.transport.write(self.conn.data_to_send())
# If there's no data left, this stream is now blocked.
if not self._outboundStreamQueues[stream]:
self.priority.block(stream)
# Also, if the stream's flow control window is exhausted, tell it
# to stop.
if self.remainingOutboundWindow(stream) <= 0:
self.streams[stream].flowControlBlocked()
self._reactor.callLater(0, self._sendPrioritisedData)
def test_basic():
"""
Test a simple one-to-one connection with WAMP pubsub.
"""
print('Running test_basic')
# Awful (but simple) way of making sure handshake is complete
handshake = deferLater(reactor, 0.5, lambda: None)
# Ordering the things to do
consumer_client_subscribe = handshake
publisher_dispatch = Deferred()
# All the events that should be triggered
producer_pubsub_received = Deferred()
consumer_pubsub_received = Deferred()
class WampProducerServerProtocol(wamp.WampServerProtocol):
def onSessionOpen(self):
self.registerForPubSub('http://example.com/mytopic')
publisher_dispatch.addCallback(lambda _: self.dispatch(
'http://example.com/mytopic',
{'a': 'b'},
))
# This one shouldn't go through
publisher_dispatch.addCallback(lambda _: self.dispatch(
'http://example.com/NOTmytopic',
{'a': 'b'},
))
class WampProducerServerFactory(ProducerMixin, wamp.WampServerFactory):
protocol = WampProducerServerProtocol
class WampProducerClientProtocol(wamp.WampClientProtocol):
def onSessionOpen(self):
self.subscribe('http://example.com/mytopic', self.onEvent)
def onEvent(self, topic, event):
try:
assert topic == 'http://example.com/mytopic'
assert event == {'a': 'b'}
producer_pubsub_received.callback(None)
except:
producer_pubsub_received.errback()
class WampProducerClientFactory(wamp.WampClientFactory):
protocol = WampProducerClientProtocol
class WampConsumerServerProtocol(wamp.WampServerProtocol):
def onSessionOpen(self):
self.registerForPubSub('http://example.com/mytopic')
class WampConsumerServerFactory(ConsumerMixin, wamp.WampServerFactory):
protocol = WampConsumerServerProtocol
class WampConsumerClientProtocol(wamp.WampClientProtocol):
def onSessionOpen(self):
consumer_client_subscribe.addCallback(lambda _: self.subscribe(
'http://example.com/mytopic',
self.onEvent,
))
consumer_client_subscribe.addCallback(lambda _: deferLater(
reactor, 0.5, lambda: None)).chainDeferred(publisher_dispatch)
def onEvent(self, topic, event):
try:
assert topic == 'http://example.com/mytopic'
assert event == {'a': 'b'}
consumer_pubsub_received.callback(None)
except:
consumer_pubsub_received.errback()
class WampConsumerClientFactory(wamp.WampClientFactory):
protocol = WampConsumerClientProtocol
consumer = ConsumerServer('localhost', 19000)
WampConsumerServerFactory.consumer = consumer
producer = ProducerClient([
('localhost', 19000),
])
WampProducerServerFactory.producer = producer
listenWS(WampProducerServerFactory('ws://localhost:19001'))
connectWS(WampProducerClientFactory('ws://localhost:19001'))
consumer_server = WampConsumerServerFactory('ws://localhost:19002')
listenWS(consumer_server)
consumer.processor = consumer_server
connectWS(WampConsumerClientFactory('ws://localhost:19002'))
return DeferredList([
consumer_client_subscribe,
publisher_dispatch,
producer_pubsub_received,
consumer_pubsub_received,
])
def start(self, node_id=None):
"""
Starts this node. This will start a node controller and then spawn new worker
processes as needed.
"""
self.log.info('Starting {personality} node {method}',
personality=self.personality.NAME,
method=hltype(Node.start))
# a configuration must have been loaded before
if not self._config:
raise Exception("No node configuration set")
# a node can only be started once for now
assert self._shutdown_complete is None
assert self._node_id is None
# get controller config/options
controller_config = self._config.get('controller', {})
controller_options = controller_config.get('options', {})
# the node ID: CLI takes precedence over config over hostname
if node_id:
self._node_id = node_id
_node_id_source = 'explicit run-time argument'
elif 'id' in controller_config:
self._node_id = controller_config['id']
_node_id_source = 'explicit configuration'
else:
self._node_id = u'{}'.format(socket.gethostname()).lower()
_node_id_source = 'hostname'
self.log.info('Node ID {node_id} set from {node_id_source}',
node_id=hlid(self._node_id),
node_id_source=_node_id_source)
# set controller process title
try:
import setproctitle
except ImportError:
self.log.warn("Warning, could not set process title (setproctitle not installed)")
else:
setproctitle.setproctitle(controller_options.get('title', 'crossbar-controller'))
# local node management router
self._router_factory = RouterFactory(self._node_id, None)
self._router_session_factory = RouterSessionFactory(self._router_factory)
rlm_config = {
'name': self._realm
}
rlm = RouterRealm(None, rlm_config)
router = self._router_factory.start_realm(rlm)
# setup global static roles
self._add_global_roles()
# always add a realm service session
cfg = ComponentConfig(self._realm)
rlm.session = (self.ROUTER_SERVICE)(cfg, router)
self._router_session_factory.add(rlm.session, authrole=u'trusted')
self.log.debug('Router service session attached [{router_service}]', router_service=qual(self.ROUTER_SERVICE))
# add the node controller singleton component
self._controller = self.NODE_CONTROLLER(self)
self._router_session_factory.add(self._controller, authrole=u'trusted')
self.log.debug('Node controller attached [{node_controller}]', node_controller=qual(self.NODE_CONTROLLER))
# add extra node controller components
self._add_extra_controller_components(controller_options)
# setup Node shutdown triggers
self._set_shutdown_triggers(controller_options)
# setup node shutdown Deferred
self._shutdown_complete = Deferred()
# startup the node personality ..
yield self.personality.Node.boot(self)
# notify systemd that we are fully up and running
try:
import sdnotify
except ImportError:
# do nothing on non-systemd platforms
pass
else:
sdnotify.SystemdNotifier().notify("READY=1")
# return a shutdown deferred which we will fire to notify the code that
# called start() - which is the main crossbar boot code
res = {
'shutdown_complete': self._shutdown_complete
}
returnValue(res)
def getProtocol(self):
d = self._get_prot = Deferred()
connectWS(self)
return d
def connectionMade(self):
self.finished = Deferred()
self.strings = ["bif", "pow", "zot"]
self.sendNoise()
def __init__(self):
# no parent init
self.deferred = Deferred()
def test_connect_replay():
"""
Test that when a producer connects, the consumer sends all existing
subscription to it.
"""
print('Running test_connect_replay')
handshake = deferLater(reactor, 0.5, lambda: None)
producer_connect = handshake
publisher_dispatch = Deferred()
# Events that should be triggered
producer_pubsub_received = Deferred()
client1_pubsub_received = Deferred()
client2_pubsub_received = Deferred()
class WampProducerServerProtocol(wamp.WampServerProtocol):
def onSessionOpen(self):
self.registerForPubSub('http://example.com/mytopic')
publisher_dispatch.addCallback(lambda _: self.dispatch(
'http://example.com/mytopic',
{'a': 'b'},
))
class WampProducerServerFactory(ProducerMixin, wamp.WampServerFactory):
protocol = WampProducerServerProtocol
class WampProducerClientProtocol(wamp.WampClientProtocol):
def onSessionOpen(self):
self.subscribe('http://example.com/mytopic', self.onEvent)
def onEvent(self, topic, event):
try:
assert topic == 'http://example.com/mytopic'
assert event == {'a': 'b'}
producer_pubsub_received.callback(None)
except:
producer_pubsub_received.errback()
class WampProducerClientFactory(wamp.WampClientFactory):
protocol = WampProducerClientProtocol
class WampConsumerServerProtocol(wamp.WampServerProtocol):
def onSessionOpen(self):
self.registerForPubSub('http://example.com/mytopic')
class WampConsumerServerFactory(ConsumerMixin, wamp.WampServerFactory):
protocol = WampConsumerServerProtocol
class WampConsumerClient1Protocol(wamp.WampClientProtocol):
def onSessionOpen(self):
self.subscribe('http://example.com/mytopic', self.onEvent)
self.subscribe('http://example.com/mytopic2', self.onEvent)
def onEvent(self, topic, event):
try:
assert topic == 'http://example.com/mytopic'
assert event == {'a': 'b'}
client1_pubsub_received.callback(None)
except:
client1_pubsub_received.errback()
class WampConsumerClient1Factory(wamp.WampClientFactory):
protocol = WampConsumerClient1Protocol
class WampConsumerClient2Protocol(wamp.WampClientProtocol):
def onSessionOpen(self):
self.subscribe('http://example.com/mytopic', self.onEvent)
def onEvent(self, topic, event):
try:
assert topic == 'http://example.com/mytopic'
assert event == {'a': 'b'}
client2_pubsub_received.callback(None)
except:
client2_pubsub_received.errback()
class WampConsumerClient2Factory(wamp.WampClientFactory):
protocol = WampConsumerClient2Protocol
consumer = ConsumerServer('localhost', 19100)
WampConsumerServerFactory.consumer = consumer
producer = ProducerClient([])
WampProducerServerFactory.producer = producer
producer_connect.addCallback(
lambda _: producer.connect('localhost', 19100))
producer_connect.addCallback(
lambda _: deferLater(reactor, 0.5, lambda: None), ).chainDeferred(
publisher_dispatch)
listenWS(WampProducerServerFactory('ws://localhost:19101'))
connectWS(WampProducerClientFactory('ws://localhost:19101'))
consumer_server = WampConsumerServerFactory('ws://localhost:19102')
listenWS(consumer_server)
consumer.processor = consumer_server
connectWS(WampConsumerClient1Factory('ws://localhost:19102'))
connectWS(WampConsumerClient2Factory('ws://localhost:19102'))
return DeferredList([
producer_pubsub_received,
client1_pubsub_received,
client2_pubsub_received,
])
def test_processExited(self):
"""
L{IProcessProtocol.processExited} is called when the child process
exits, even if file descriptors associated with the child are still
open.
"""
exited = Deferred()
allLost = Deferred()
lost = []
class Waiter(ProcessProtocol):
def childDataReceived(self, fd, data):
msg('childDataReceived(%d, %r)' % (fd, data))
def childConnectionLost(self, childFD):
msg('childConnectionLost(%d)' % (childFD, ))
lost.append(childFD)
if len(lost) == 3:
allLost.callback(None)
def processExited(self, reason):
msg('processExited(%r)' % (reason, ))
# See test_processExitedWithSignal
exited.callback([reason])
self.transport.loseConnection()
reactor = self.buildReactor()
reactor.callWhenRunning(
reactor.spawnProcess,
Waiter(),
pyExe, [
pyExe, b"-u", b"-m", self.keepStdioOpenProgram, b"child",
self.keepStdioOpenArg
],
env=properEnv,
usePTY=self.usePTY)
def cbExited(args):
failure, = args
failure.trap(ProcessDone)
msg('cbExited; lost = %s' % (lost, ))
self.assertEqual(lost, [])
return allLost
exited.addCallback(cbExited)
def cbAllLost(ignored):
self.assertEqual(set(lost), set([0, 1, 2]))
exited.addCallback(cbAllLost)
exited.addErrback(err)
exited.addCallback(lambda ign: reactor.stop())
self.runReactor(reactor)
def read_framed_message(self, response): finished = Deferred() response.deliverBody(AvroProtocol(finished)) return finished
def test_processExitedWithSignal(self):
"""
The C{reason} argument passed to L{IProcessProtocol.processExited} is a
L{ProcessTerminated} instance if the child process exits with a signal.
"""
sigName = 'TERM'
sigNum = getattr(signal, 'SIG' + sigName)
exited = Deferred()
source = (
b"import sys\n"
# Talk so the parent process knows the process is running. This is
# necessary because ProcessProtocol.makeConnection may be called
# before this process is exec'd. It would be unfortunate if we
# SIGTERM'd the Twisted process while it was on its way to doing
# the exec.
b"sys.stdout.write('x')\n"
b"sys.stdout.flush()\n"
b"sys.stdin.read()\n")
class Exiter(ProcessProtocol):
def childDataReceived(self, fd, data):
msg('childDataReceived(%d, %r)' % (fd, data))
self.transport.signalProcess(sigName)
def childConnectionLost(self, fd):
msg('childConnectionLost(%d)' % (fd, ))
def processExited(self, reason):
msg('processExited(%r)' % (reason, ))
# Protect the Deferred from the failure so that it follows
# the callback chain. This doesn't use the errback chain
# because it wants to make sure reason is a Failure. An
# Exception would also make an errback-based test pass, and
# that would be wrong.
exited.callback([reason])
def processEnded(self, reason):
msg('processEnded(%r)' % (reason, ))
reactor = self.buildReactor()
reactor.callWhenRunning(reactor.spawnProcess,
Exiter(),
pyExe, [pyExe, b"-c", source],
usePTY=self.usePTY)
def cbExited(args):
failure, = args
# Trapping implicitly verifies that it's a Failure (rather than
# an exception) and explicitly makes sure it's the right type.
failure.trap(ProcessTerminated)
err = failure.value
if platform.isWindows():
# Windows can't really /have/ signals, so it certainly can't
# report them as the reason for termination. Maybe there's
# something better we could be doing here, anyway? Hard to
# say. Anyway, this inconsistency between different platforms
# is extremely unfortunate and I would remove it if I
# could. -exarkun
self.assertIsNone(err.signal)
self.assertEqual(err.exitCode, 1)
else:
self.assertEqual(err.signal, sigNum)
self.assertIsNone(err.exitCode)
exited.addCallback(cbExited)
exited.addErrback(err)
exited.addCallback(lambda ign: reactor.stop())
self.runReactor(reactor)
def __init__(self):
self.deferred = Deferred()
def _runCallbacks(self):
if self.canceled:
self.callbacks = []
return
Deferred._runCallbacks(self)
def test_childConnectionLost(self):
"""
L{IProcessProtocol.childConnectionLost} is called each time a file
descriptor associated with a child process is closed.
"""
connected = Deferred()
lost = {0: Deferred(), 1: Deferred(), 2: Deferred()}
class Closer(ProcessProtocol):
def makeConnection(self, transport):
connected.callback(transport)
def childConnectionLost(self, childFD):
lost[childFD].callback(None)
target = b"twisted.internet.test.process_loseconnection"
reactor = self.buildReactor()
reactor.callWhenRunning(reactor.spawnProcess,
Closer(),
pyExe, [pyExe, b"-m", target],
env=properEnv,
usePTY=self.usePTY)
def cbConnected(transport):
transport.write(b'2\n')
return lost[2].addCallback(lambda ign: transport)
connected.addCallback(cbConnected)
def lostSecond(transport):
transport.write(b'1\n')
return lost[1].addCallback(lambda ign: transport)
connected.addCallback(lostSecond)
def lostFirst(transport):
transport.write(b'\n')
connected.addCallback(lostFirst)
connected.addErrback(err)
def cbEnded(ignored):
reactor.stop()
connected.addCallback(cbEnded)
self.runReactor(reactor)
def deferMe():
d = Deferred()
d.addErrback(deferMeMore)
return d
class DatabaseProtocol(Protocol):
WAITING_FOR_HANDSHAKE = 0
READY = 1
def __init__(self, factory):
self.factory = factory
self.state = DatabaseProtocol.WAITING_FOR_HANDSHAKE
self._handlers = {
DatabaseProtocol.WAITING_FOR_HANDSHAKE: self._handleHandshake,
DatabaseProtocol.READY: self._handleResponse
}
self.buf = bytes()
self.buf_expected_length = 0
self.buf_token = None
self.wait_for_handshake = Deferred()
self._open = True
def connectionMade(self):
# Send immediately the handshake.
self.factory.handshake.reset()
self.transport.write(self.factory.handshake.next_message(None))
# Defer a timer which will callback when timed out and errback the
# wait_for_handshake. Otherwise, it will be cancelled in
# handleHandshake.
self._timeout_defer = reactor.callLater(self.factory.timeout,
self._handleHandshakeTimeout)
def connectionLost(self, reason):
self._open = False
def _handleHandshakeTimeout(self):
# If we are here, we failed to do the handshake before the timeout.
# We close the connection and raise an ReqlTimeoutError in the
# wait_for_handshake deferred.
self._open = False
self.transport.loseConnection()
self.wait_for_handshake.errback(ReqlTimeoutError())
def _handleHandshake(self, data):
try:
self.buf += data
while True:
end_index = self.buf.find(b'\0')
if end_index != -1:
response = self.buf[:end_index]
self.buf = self.buf[end_index + 1:]
request = self.factory.handshake.next_message(response)
if request is None:
# We're now ready to work with real data.
self.state = DatabaseProtocol.READY
# We cancel the scheduled timeout.
self._timeout_defer.cancel()
# We callback our wait_for_handshake.
self.wait_for_handshake.callback(None)
elif request != "":
self.transport.write(request)
else:
break
except Exception as e:
self.wait_for_handshake.errback(e)
def _handleResponse(self, data):
# If we have more than one response, we should handle all of them.
self.buf += data
while True:
# 1. Read the header, until we read the length of the awaited payload.
if self.buf_expected_length == 0:
if len(self.buf) >= 12:
token, length = struct.unpack('<qL', self.buf[:12])
self.buf_token = token
self.buf_expected_length = length
self.buf = self.buf[12:]
else:
# We quit the function, it is impossible to have read the
# entire payload at this point.
return
# 2. Buffer the data, until the size of the data match the expected
# length provided by the header.
if len(self.buf) < self.buf_expected_length:
return
self.factory.response_handler(self.buf_token,
self.buf[:self.buf_expected_length])
self.buf = self.buf[self.buf_expected_length:]
self.buf_token = None
self.buf_expected_length = 0
def dataReceived(self, data):
try:
if self._open:
self._handlers[self.state](data)
except Exception as e:
raise ReqlDriverError(
'Driver failed to handle received data.'
'Error: {exc}. Dropping the connection.'.format(exc=str(e)))
self.transport.loseConnection()
def wait(self, seconds, result=None):
"""Returns a deferred that will be fired later"""
d = Deferred()
reactor.callLater(seconds, d.callback, result)
return d
def __init__(self):
Deferred.__init__(self)
self.canceled = 0
