def test_wait_for_brokers(self): """ The L{Deferred} returned by L{Tub.stopService} fires only after the L{Broker} connections belonging to the L{Tub} have disconnected. """ tub = Tub() tub.startService() another_tub = Tub() another_tub.startService() brokers = list(tub.brokerClass(None) for i in range(3)) for n, b in enumerate(brokers): b.makeConnection(StringTransport()) ref = SturdyRef(encode_furl(another_tub.tubID, [], str(n))) tub.brokerAttached(ref, b, isClient=(n % 2) == 1) stopping = tub.stopService() d = flushEventualQueue() def event(ignored): self.assertNoResult(stopping) for b in brokers: b.connectionLost(failure.Failure(Exception("Connection lost"))) return flushEventualQueue() d.addCallback(event) def connectionsLost(ignored): self.successResultOf(stopping) d.addCallback(connectionsLost) return d
def test_wait_for_brokers(self): """ The L{Deferred} returned by L{Tub.stopService} fires only after the L{Broker} connections belonging to the L{Tub} have disconnected. """ tub = Tub() tub.startService() another_tub = Tub() another_tub.startService() brokers = list(tub.brokerClass(None) for i in range(3)) for n, b in enumerate(brokers): b.makeConnection(StringTransport()) ref = SturdyRef(encode_furl(another_tub.tubID, [], str(n))) tub.brokerAttached(ref, b, isClient=(n % 2)==1) stopping = tub.stopService() d = flushEventualQueue() def event(ignored): self.assertNoResult(stopping) for b in brokers: b.connectionLost(failure.Failure(Exception("Connection lost"))) return flushEventualQueue() d.addCallback(event) def connectionsLost(ignored): self.successResultOf(stopping) d.addCallback(connectionsLost) return d
def test_retry(self): tubC = Tub(certData=self.tubB.getCertData()) connects = [] target = HelperTarget("bob") url = self.tubB.registerReference(target, "target") portb = self.tub_ports[1] d1 = defer.Deferred() notifiers = [d1] self.services.remove(self.tubB) # This will fail, since tubB is not listening anymore. Wait until it's # moved to the "waiting" state. yield self.tubB.stopService() rc = self.tubA.connectTo(url, self._connected, notifiers, connects) yield self.poll(lambda: rc.getReconnectionInfo().state == "waiting") self.failUnlessEqual(len(connects), 0) # now start tubC listening on the same port that tubB used to, which # should allow the connection to complete (since they both use the same # certData) self.services.append(tubC) tubC.startService() tubC.listenOn("tcp:%d:interface=127.0.0.1" % portb) tubC.setLocation("tcp:127.0.0.1:%d" % portb) url2 = tubC.registerReference(target, "target") assert url2 == url yield d1 self.failUnlessEqual(len(connects), 1) rc.stopConnecting()
def do_remote_command(command, *args, **kwargs): client = AnyConsensoProcess() client.start() furl = client.furl() tub = Tub() tub.startService() def got_error(err): print "Error while calling command remotely", err reactor.stop() def got_result(res): print(res) def got_remote(remote): d = remote.callRemote(command, *args, **kwargs) d.addCallback(got_result) d.addCallback(lambda res: reactor.stop()) d.addErrback(got_error) return d d = tub.getReference(furl) d.addCallback(got_remote) d.addErrback(got_error) reactor.run()
def testAuthenticated(self): url, portnum = self.makeServer() client = Tub() client.startService() self.services.append(client) d = client.getReference(url) return d
class LogTail: def __init__(self, options): self.options = options def run(self, target_furl): target_tubid = SturdyRef(target_furl).getTubRef().getTubID() d = fireEventually(target_furl) d.addCallback(self.start, target_tubid) d.addErrback(self._error) print("starting..") reactor.run() def _error(self, f): print("ERROR", f) reactor.stop() def start(self, target_furl, target_tubid): print("Connecting..") self._tub = Tub() self._tub.startService() self._tub.connectTo(target_furl, self._got_logpublisher, target_tubid) def _got_logpublisher(self, publisher, target_tubid): d = publisher.callRemote("get_pid") def _announce(pid_or_failure): if isinstance(pid_or_failure, int): print("Connected (to pid %d)" % pid_or_failure) return pid_or_failure else: # the logport is probably foolscap-0.2.8 or earlier and # doesn't offer get_pid() print("Connected (unable to get pid)") return None d.addBoth(_announce) publisher.notifyOnDisconnect(self._lost_logpublisher) lp = LogPrinter(self.options, target_tubid) def _ask_for_versions(pid): d = publisher.callRemote("get_versions") d.addCallback(lp.got_versions, pid) return d d.addCallback(_ask_for_versions) catch_up = bool(self.options["catch-up"]) if catch_up: d.addCallback( lambda res: publisher.callRemote("subscribe_to_all", lp, True)) else: # provide compatibility with foolscap-0.2.4 and earlier, which # didn't accept a catchup= argument d.addCallback( lambda res: publisher.callRemote("subscribe_to_all", lp)) d.addErrback(self._error) return d def _lost_logpublisher(publisher): print("Disconnected")
def setUp(self): if not crypto_available: raise unittest.SkipTest("crypto not available") self.services = [] for i in range(self.num_services): s = Tub() s.startService() self.services.append(s)
def create_tub(self, basedir): os.makedirs(basedir) tubfile = os.path.join(basedir, "tub.pem") tub = Tub(certFile=tubfile) tub.setOption("expose-remote-exception-types", False) tub.startService() self.addCleanup(tub.stopService) return tub
def testHalfAuthenticated2(self): if not crypto_available: raise unittest.SkipTest("crypto not available") url, portnum = self.makeServer(False) client = Tub() client.startService() self.services.append(client) d = client.getReference(url) return d
def _testNoConnection_1(self, res, url): self.services.remove(self.tub) client = Tub() client.startService() self.services.append(client) d = client.getReference(url) d.addCallbacks(lambda res: self.fail("this is supposed to fail"), self._testNoConnection_fail) return d
def createSpecificServer(self, certData, negotiationClass=negotiate.Negotiation): tub = Tub(certData=certData) tub.negotiationClass = negotiationClass tub.startService() self.services.append(tub) l = tub.listenOn("tcp:0") tub.setLocation("127.0.0.1:%d" % l.getPortnum()) target = Target() return tub, target, tub.registerReference(target), l.getPortnum()
class LogTail: def __init__(self, options): self.options = options def run(self, target_furl): target_tubid = SturdyRef(target_furl).getTubRef().getTubID() d = fireEventually(target_furl) d.addCallback(self.start, target_tubid) d.addErrback(self._error) print "starting.." reactor.run() def _error(self, f): print "ERROR", f reactor.stop() def start(self, target_furl, target_tubid): print "Connecting.." self._tub = Tub() self._tub.startService() self._tub.connectTo(target_furl, self._got_logpublisher, target_tubid) def _got_logpublisher(self, publisher, target_tubid): d = publisher.callRemote("get_pid") def _announce(pid_or_failure): if isinstance(pid_or_failure, int): print "Connected (to pid %d)" % pid_or_failure return pid_or_failure else: # the logport is probably foolscap-0.2.8 or earlier and # doesn't offer get_pid() print "Connected (unable to get pid)" return None d.addBoth(_announce) publisher.notifyOnDisconnect(self._lost_logpublisher) lp = LogPrinter(self.options, target_tubid) def _ask_for_versions(pid): d = publisher.callRemote("get_versions") d.addCallback(lp.got_versions, pid) return d d.addCallback(_ask_for_versions) catch_up = bool(self.options["catch-up"]) if catch_up: d.addCallback(lambda res: publisher.callRemote("subscribe_to_all", lp, True)) else: # provide compatibility with foolscap-0.2.4 and earlier, which # didn't accept a catchup= argument d.addCallback(lambda res: publisher.callRemote("subscribe_to_all", lp)) d.addErrback(self._error) return d def _lost_logpublisher(publisher): print "Disconnected"
def testClientTimeout(self): portnum = self.makeNullServer() # lower the connection timeout to 2 seconds client = Tub(_test_options={'connect_timeout': 1}) client.startService() self.services.append(client) url = "pb://[email protected]:%d/target" % portnum d = client.getReference(url) d.addCallbacks(lambda res: self.fail("hey! this is supposed to fail"), lambda f: f.trap(tokens.NegotiationError)) return d
def createSpecificServer(self, certData, negotiationClass=negotiate.Negotiation): tub = Tub(certData=certData) tub.negotiationClass = negotiationClass tub.startService() self.services.append(tub) portnum = allocate_tcp_port() tub.listenOn("tcp:%d:interface=127.0.0.1" % portnum) tub.setLocation("127.0.0.1:%d" % portnum) target = Target() return tub, target, tub.registerReference(target), portnum
def createDuplicateServer(self, oldtub): tub = Tub(certData=oldtub.getCertData()) tub.startService() self.services.append(tub) tub.incarnation = oldtub.incarnation tub.incarnation_string = oldtub.incarnation_string tub.slave_table = oldtub.slave_table.copy() tub.master_table = oldtub.master_table.copy() l = tub.listenOn("tcp:0") tub.setLocation("127.0.0.1:%d" % l.getPortnum()) target = Target() return tub, target, tub.registerReference(target), l.getPortnum()
def testFuture3(self): # same as testFuture1, but it is the listening server that # understands [1,2] url, portnum = self.makeSpecificServer(certData_high, NegotiationVbig) client = Tub(certData=certData_low) client.startService() self.services.append(client) d = client.getReference(url) def _check_version(rref): ver = rref.tracker.broker._banana_decision_version self.failUnlessEqual(ver, MAX_HANDLED_VERSION) d.addCallback(_check_version) return d
def testVersusHTTPServerAuthenticated(self): portnum = self.makeHTTPServer() client = Tub() client.startService() self.services.append(client) url = "pb://%[email protected]:%d/target" % (tubid_low, portnum) d = client.getReference(url) d.addCallbacks(lambda res: self.fail("this is supposed to fail"), lambda f: f.trap(BananaError)) # the HTTP server needs a moment to notice that the connection has # gone away. Without this, trial flunks the test because of the # leftover HTTP server socket. d.addCallback(self.stall, 1) return d
def testFuture2(self): # same as before, but the connecting Tub will have the higher tubID, # and thus make the negotiation decision url, portnum = self.makeSpecificServer(certData_low) # the client client = Tub(certData=certData_high) client.negotiationClass = NegotiationVbig client.startService() self.services.append(client) d = client.getReference(url) def _check_version(rref): ver = rref.tracker.broker._banana_decision_version self.failUnlessEqual(ver, MAX_HANDLED_VERSION) d.addCallback(_check_version) return d
def testTooFarInFuture4(self): # same as testTooFarInFuture2, but it is the listening server which # only understands [2] url, portnum = self.makeSpecificServer(certData_low, NegotiationVbigOnly) client = Tub(certData=certData_high) client.startService() self.services.append(client) d = client.getReference(url) def _oops_succeeded(rref): self.fail("hey! this is supposed to fail") def _check_failure(f): f.trap(tokens.NegotiationError, tokens.RemoteNegotiationError) d.addCallbacks(_oops_succeeded, _check_failure) return d
def makeTubs(self, numTubs, mangleLocation=None): self.services = [] self.tub_ports = [] for i in range(numTubs): t = Tub() t.startService() self.services.append(t) portnum = allocate_tcp_port() self.tub_ports.append(portnum) t.listenOn("tcp:%d:interface=127.0.0.1" % portnum) location = "127.0.0.1:%d" % portnum if mangleLocation: location = mangleLocation(portnum) t.setLocation(location) return self.services
def testTooFarInFuture2(self): # same as before, but the connecting Tub will have the higher tubID, # and thus make the negotiation decision url, portnum = self.makeSpecificServer(certData_low) client = Tub(certData=certData_high) client.negotiationClass = NegotiationVbigOnly client.startService() self.services.append(client) d = client.getReference(url) def _oops_succeeded(rref): self.fail("hey! this is supposed to fail") def _check_failure(f): f.trap(tokens.NegotiationError, tokens.RemoteNegotiationError) d.addCallbacks(_oops_succeeded, _check_failure) return d
def test_doublestop(self): tub = Tub() tub.startService() d = tub.stopService() d.addCallback(lambda res: self.shouldFail( RuntimeError, "test_doublestop_startService", "Sorry, but Tubs cannot be restarted", tub.startService)) d.addCallback(lambda res: self.shouldFail( RuntimeError, "test_doublestop_getReference", "Sorry, but this Tub has been shut down", tub.getReference, "furl") ) d.addCallback(lambda res: self.shouldFail( RuntimeError, "test_doublestop_connectTo", "Sorry, but this Tub has been shut down", tub.connectTo, "furl", None)) return d
def run_command(config): c = dispatch_table[config.subCommand]() tub = Tub() try: from twisted.internet import reactor from twisted.internet.endpoints import clientFromString from foolscap.connections import tor CONTROL = os.environ.get("FOOLSCAP_TOR_CONTROL_PORT", "") SOCKS = os.environ.get("FOOLSCAP_TOR_SOCKS_PORT", "") if CONTROL: h = tor.control_endpoint(clientFromString(reactor, CONTROL)) tub.addConnectionHintHandler("tor", h) elif SOCKS: h = tor.socks_endpoint(clientFromString(reactor, SOCKS)) tub.addConnectionHintHandler("tor", h) #else: # h = tor.default_socks() # tub.addConnectionHintHandler("tor", h) except ImportError: pass d = defer.succeed(None) d.addCallback(lambda _ign: tub.startService()) d.addCallback(lambda _ign: tub.getReference(config.furl)) d.addCallback(c.run, config.subOptions) # might provide tub here d.addBoth(lambda res: tub.stopService().addCallback(lambda _ign: res)) return d
class Failed(unittest.TestCase): def setUp(self): self.services = [] def tearDown(self): d = defer.DeferredList([s.stopService() for s in self.services]) d.addCallback(flushEventualQueue) return d @defer.inlineCallbacks def test_bad_hints(self): self.tubA = Tub() self.tubA.startService() self.services.append(self.tubA) self.tubB = Tub() self.tubB.startService() self.services.append(self.tubB) portnum = allocate_tcp_port() self.tubB.listenOn("tcp:%d:interface=127.0.0.1" % portnum) bad1 = "no-colon" bad2 = "unknown:foo" bad3 = "tcp:300.300.300.300:333" self.tubB.setLocation(bad1, bad2, bad3) target = HelperTarget("bob") url = self.tubB.registerReference(target) rc = self.tubA.connectTo(url, None) ri = rc.getReconnectionInfo() self.assertEqual(ri.state, "connecting") d = defer.Deferred() reactor.callLater(1.0, d.callback, None) yield d # now look at the details ri = rc.getReconnectionInfo() self.assertEqual(ri.state, "waiting") ci = ri.connectionInfo self.assertEqual(ci.connected, False) self.assertEqual(ci.winningHint, None) s = ci.connectorStatuses self.assertEqual(set(s.keys()), set([bad1, bad2, bad3])) self.assertEqual(s[bad1], "bad hint: no colon") self.assertEqual(s[bad2], "bad hint: no handler registered") self.assertIn("DNS lookup failed", s[bad3]) ch = ci.connectionHandlers self.assertEqual(ch, {bad2: None, bad3: "tcp"})
def _testPersist_1(self, res, s1, s2, t1, public_url, port): self.services.remove(s1) s3 = Tub(certData=s1.getCertData()) s3.startService() self.services.append(s3) t2 = Target() newport = allocate_tcp_port() s3.listenOn("tcp:%d:interface=127.0.0.1" % newport) s3.setLocation("127.0.0.1:%d" % newport) s3.registerReference(t2, "name") # now patch the URL to replace the port number newurl = re.sub(":%d/" % port, ":%d/" % newport, public_url) d = s2.getReference(newurl) d.addCallback(lambda rr: rr.callRemote("add", a=1, b=2)) d.addCallback(self.failUnlessEqual, 3) d.addCallback(self._testPersist_2, t1, t2) return d
def test1(self): # Two FURLs pointing at the same Tub should share a connection. This # basically exercises TubRef.__cmp__ . furl1, furl2 = self.makeServers() client = Tub(certData_low) client.startService() self.services.append(client) d = client.getReference(furl1) rrefs = [] d.addCallback(rrefs.append) d.addCallback(lambda _: client.getReference(furl2)) d.addCallback(rrefs.append) def _check(_): self.failUnlessIdentical(rrefs[0].tracker.broker, rrefs[1].tracker.broker) d.addCallback(_check) return d
def run_command(config): c = dispatch_table[config.subCommand]() tub = Tub() d = defer.succeed(None) d.addCallback(lambda _ign: tub.startService()) d.addCallback(lambda _ign: tub.getReference(config.furl)) d.addCallback(c.run, config.subOptions) # might provide tub here d.addBoth(lambda res: tub.stopService().addCallback(lambda _ign: res)) return d
def testFuture1(self): # when a peer that understands version=[1] that connects to a peer # that understands version=[1,2], they should pick version=1 # the listening Tub will have the higher tubID, and thus make the # negotiation decision url, portnum = self.makeSpecificServer(certData_high) # the client client = Tub(certData=certData_low) client.negotiationClass = NegotiationVbig client.startService() self.services.append(client) d = client.getReference(url) def _check_version(rref): ver = rref.tracker.broker._banana_decision_version self.failUnlessEqual(ver, MAX_HANDLED_VERSION) d.addCallback(_check_version) return d
class FlappCommand(object): def __init__(self, furlfile): self.flappclient_args = ["-f", furlfile, "run-command"] options = ClientOptions() options.parseOptions(self.flappclient_args) self.furl = options.furl self.tub = Tub() self.rref = None self.d = defer.succeed(None) def start(self): self.d.addCallback(lambda ign: self.tub.startService()) self.d.addCallback(lambda ign: self.tub.getReference(self.furl)) done = defer.Deferred() def _got_rref(rref): self.rref = rref done.callback(None) self.d.addCallbacks(_got_rref, done.errback) return done def run(self, content, stdout, stderr, when_done, when_failed): assert self.rref is not None options = ClientOptions() options.stdout = stdout options.stderr = stderr options.parseOptions(self.flappclient_args) def stdio(proto): proto.dataReceived(content) proto.connectionLost("EOF") options.subOptions.stdio = stdio options.subOptions.stdout = stdout options.subOptions.stderr = stderr def _go(ign): print >>stdout, "Starting..." return RunCommand().run(self.rref, options.subOptions) def _done(rc): if rc == 0: when_done() else: print >>stdout, "Command failed with exit code %r." % (rc,) when_failed() def _error(f): print >>stdout, str(f) when_failed() def _recover(f): try: print >>stderr, str(f) except Exception: print >>stderr, "something weird" self.d.addCallback(_go) self.d.addCallbacks(_done, _error) self.d.addErrback(_recover)
def testTooFarInFuture1(self): # when a peer that understands version=[1] that connects to a peer # that only understands version=[2], they should fail to negotiate # the listening Tub will have the higher tubID, and thus make the # negotiation decision url, portnum = self.makeSpecificServer(certData_high) # the client client = Tub(certData=certData_low) client.negotiationClass = NegotiationVbigOnly client.startService() self.services.append(client) d = client.getReference(url) def _oops_succeeded(rref): self.fail("hey! this is supposed to fail") def _check_failure(f): f.trap(tokens.NegotiationError, tokens.RemoteNegotiationError) d.addCallbacks(_oops_succeeded, _check_failure) return d
def test_doublestop(self): tub = Tub() tub.startService() d = tub.stopService() d.addCallback(lambda res: self.shouldFail(RuntimeError, "test_doublestop_startService", "Sorry, but Tubs cannot be restarted", tub.startService)) d.addCallback(lambda res: self.shouldFail(RuntimeError, "test_doublestop_getReference", "Sorry, but this Tub has been shut down", tub.getReference, "furl")) d.addCallback(lambda res: self.shouldFail(RuntimeError, "test_doublestop_connectTo", "Sorry, but this Tub has been shut down", tub.connectTo, "furl", None)) return d
def test_lose_and_retry(self): tubC = Tub(self.tubB.getCertData()) connects = [] d1 = defer.Deferred() d2 = defer.Deferred() notifiers = [d1, d2] target = HelperTarget("bob") url = self.tubB.registerReference(target, "target") portb = self.tub_ports[1] rc = self.tubA.connectTo(url, self._connected, notifiers, connects) yield d1 self.assertEqual(rc.getReconnectionInfo().state, "connected") # we are now connected to tubB. Shut it down to force a disconnect. self.services.remove(self.tubB) yield self.tubB.stopService() # wait for at least one retry yield self.poll(lambda: rc.getReconnectionInfo().state == "waiting") # wait a few seconds more to give the Reconnector a chance to try and # fail a few times. It isn't easy to catch the "connecting" state since # the target is local and the kernel knows that it's not listening. # TODO: add an internal retry counter to the Reconnector that we can # poll for tests. yield self.stall(2) # now start tubC listening on the same port that tubB used to, # which should allow the connection to complete (since they both # use the same certData) self.services.append(tubC) tubC.startService() tubC.listenOn("tcp:%d:interface=127.0.0.1" % portb) tubC.setLocation("tcp:127.0.0.1:%d" % portb) url2 = tubC.registerReference(target, "target") assert url2 == url # this will fire when the second connection has been made yield d2 self.failUnlessEqual(len(connects), 2) rc.stopConnecting()
class FlappCommand(object): def __init__(self, furl): self.flappclient_args = ["--furl", furl, "run-command"] options = ClientOptions() options.parseOptions(self.flappclient_args) self.furl = options.furl self.tub = Tub() self.rref = None self.d = defer.succeed(None) def start(self): self.d.addCallback(lambda ign: self.tub.startService()) self.d.addCallback(lambda ign: self.tub.getReference(self.furl)) done = defer.Deferred() def _got_rref(rref): self.rref = rref done.callback(None) def _failed(f): done.errback(f) return f self.d.addCallbacks(_got_rref, _failed) return done def run(self, content, log): assert isinstance(content, bytes), (`content`, type(content)) assert self.rref is not None options = ClientOptions() options.parseOptions(self.flappclient_args) def stdio(proto): # This value is being sent to the stdin of the flapp. proto.dataReceived(content) proto.connectionLost("EOF") options.subOptions.stdio = stdio # These are not used. options.subOptions.stdout = None options.subOptions.stderr = None print >>log, "Starting command." self.d = RunCommand().run(self.rref, options.subOptions) def _log_return_code(rc): print >>log, "Command completed with exit code %r" % (rc,) def _log_failure(f): print >>log, "Command failed with %r" % (f,) self.d.addCallbacks(_log_return_code, _log_failure) return self.d
class FlappCommand(object): def __init__(self, furlfile): self.flappclient_args = ["-f", furlfile, "run-command"] options = ClientOptions() options.parseOptions(self.flappclient_args) self.furl = options.furl self.tub = Tub() self.rref = None self.d = defer.succeed(None) def start(self): self.d.addCallback(lambda ign: self.tub.startService()) self.d.addCallback(lambda ign: self.tub.getReference(self.furl)) done = defer.Deferred() def _got_rref(rref): self.rref = rref done.callback(None) def _failed(f): done.errback(f) return f self.d.addCallbacks(_got_rref, _failed) return done def run(self, content, log): assert isinstance(content, bytes), (`content`, type(content)) assert self.rref is not None options = ClientOptions() options.parseOptions(self.flappclient_args) def stdio(proto): # This value is being sent to the stdin of the flapp. proto.dataReceived(content) proto.connectionLost("EOF") options.subOptions.stdio = stdio # These are not used. options.subOptions.stdout = None options.subOptions.stderr = None print >>log, "Starting command." self.d = RunCommand().run(self.rref, options.subOptions) def _log_return_code(rc): print >>log, "Command completed with exit code %r" % (rc,) def _log_failure(f): print >>log, "Command failed with %r" % (f,) self.d.addCallbacks(_log_return_code, _log_failure) return self.d
class CancelPendingDeliveries(unittest.TestCase, StallMixin): def tearDown(self): dl = [defer.succeed(None)] if self.tubA.running: dl.append(defer.maybeDeferred(self.tubA.stopService)) if self.tubB.running: dl.append(defer.maybeDeferred(self.tubB.stopService)) d = defer.DeferredList(dl) d.addCallback(flushEventualQueue) return d def test_cancel_pending_deliveries(self): # when a Tub is stopped, any deliveries that were pending should be # discarded. TubA sends remote_one+remote_two (and we hope they # arrive in the same chunk). TubB responds to remote_one by shutting # down. remote_two should be discarded. The bug was that remote_two # would cause an unhandled error on the TubB side. self.tubA = Tub() self.tubB = Tub() self.tubA.startService() self.tubB.startService() self.tubB.listenOn("tcp:0") d = self.tubB.setLocationAutomatically() r = Receiver(self.tubB) d.addCallback(lambda res: self.tubB.registerReference(r)) d.addCallback(lambda furl: self.tubA.getReference(furl)) def _go(rref): # we want these two to get sent and received in the same hunk rref.callRemoteOnly("one") rref.callRemoteOnly("two") return r.done_d d.addCallback(_go) # let remote_two do its log.err before we move on to the next test d.addCallback(self.stall, 1.0) return d