def test_unreachable_gift(self):
        client_tub = Tub()
        client_tub.setServiceParent(self.s)
        server_tub = Tub()
        server_tub.setServiceParent(self.s)
        recipient_tub = Tub()
        recipient_tub.setServiceParent(self.s)

        portnum = allocate_tcp_port()
        server_tub.listenOn("tcp:%d:interface=127.0.0.1" % portnum)
        server_tub.setLocation("tcp:127.0.0.1:%d" % portnum)
        s = Receiver() # no FURL, not directly reachable
        r = Receiver()
        furl = server_tub.registerReference(r)

        d = client_tub.getReference(furl)
        d.addCallback(lambda rref: rref.callRemote("call", s))
        d.addCallback(lambda res: self.failUnlessEqual(res, 1))
        d.addCallback(lambda _: recipient_tub.getReference(furl))
        # when server_tub tries to send the lame 's' rref to recipient_tub,
        # the RemoteReferenceTracker won't have a FURL, so it will be
        # serialized as a (their-reference furl="") sequence. Then
        # recipient_tub will try to resolve it, and will throw a
        # NoLocationHintsError. It might be more natural to send
        # (their-reference furl=None), but the constraint schema on
        # their-references forbids non-strings. It might also seem
        # appropriate to raise a Violation (i.e. server_tub is bad for trying
        # to send it, rather than foisting the problem off to recipient_tub),
        # but that causes the connection explode and fall out of sync.
        d.addCallback(lambda rref:
                      self.shouldFail(NoLocationHintsError, "gift_me", None,
                                      rref.callRemote, "gift_me"))
        return d
Beispiel #2
0
    def test_unreachable_gift(self):
        client_tub = Tub()
        client_tub.setServiceParent(self.s)
        server_tub = Tub()
        server_tub.setServiceParent(self.s)
        recipient_tub = Tub()
        recipient_tub.setServiceParent(self.s)

        portnum = allocate_tcp_port()
        server_tub.listenOn("tcp:%d:interface=127.0.0.1" % portnum)
        server_tub.setLocation("tcp:127.0.0.1:%d" % portnum)
        s = Receiver()  # no FURL, not directly reachable
        r = Receiver()
        furl = server_tub.registerReference(r)

        d = client_tub.getReference(furl)
        d.addCallback(lambda rref: rref.callRemote("call", s))
        d.addCallback(lambda res: self.assertEqual(res, 1))
        d.addCallback(lambda _: recipient_tub.getReference(furl))
        # when server_tub tries to send the lame 's' rref to recipient_tub,
        # the RemoteReferenceTracker won't have a FURL, so it will be
        # serialized as a (their-reference furl="") sequence. Then
        # recipient_tub will try to resolve it, and will throw a
        # NoLocationHintsError. It might be more natural to send
        # (their-reference furl=None), but the constraint schema on
        # their-references forbids non-strings. It might also seem
        # appropriate to raise a Violation (i.e. server_tub is bad for trying
        # to send it, rather than foisting the problem off to recipient_tub),
        # but that causes the connection explode and fall out of sync.
        d.addCallback(lambda rref: self.shouldFail(
            NoLocationHintsError, "gift_me", None, rref.callRemote, "gift_me"))
        return d
Beispiel #3
0
 def testAuthenticated(self):
     url, portnum = self.makeServer()
     client = Tub()
     client.startService()
     self.services.append(client)
     d = client.getReference(url)
     return d
Beispiel #4
0
    def test_unreachable_client(self):
        # A "client-only" Tub has no location set. It should still be
        # possible to connect to objects in other (location-bearing) server
        # Tubs, and objects in the client Tub can still be sent to (and used
        # by) the server Tub.

        client_tub = Tub()
        client_tub.setServiceParent(self.s)
        server_tub = Tub()
        server_tub.setServiceParent(self.s)

        portnum = allocate_tcp_port()
        server_tub.listenOn("tcp:%d:interface=127.0.0.1" % portnum)
        server_tub.setLocation("tcp:127.0.0.1:%d" % portnum)
        s = Receiver()  # no FURL, not directly reachable
        r = Receiver()
        furl = server_tub.registerReference(r)

        d = client_tub.getReference(furl)
        d.addCallback(lambda rref: rref.callRemote("call", s))
        d.addCallback(lambda res: self.assertEqual(res, 1))
        d.addCallback(lambda _: self.assertNotEqual(r.obj, None))

        def _inspect_obj(_):
            self.assertEqual(r.obj.getSturdyRef().getURL(), None)

        d.addCallback(_inspect_obj)
        d.addCallback(lambda _: r.obj.callRemote("call", 2))
        d.addCallback(lambda _: self.assertEqual(s.obj, 2))
        return d
Beispiel #5
0
def run_command(config):
    c = dispatch_table[config.subCommand]()
    tub = Tub()
    try:
        from twisted.internet import reactor
        from twisted.internet.endpoints import clientFromString
        from foolscap.connections import tor
        CONTROL = os.environ.get("FOOLSCAP_TOR_CONTROL_PORT", "")
        SOCKS = os.environ.get("FOOLSCAP_TOR_SOCKS_PORT", "")
        if CONTROL:
            h = tor.control_endpoint(clientFromString(reactor, CONTROL))
            tub.addConnectionHintHandler("tor", h)
        elif SOCKS:
            h = tor.socks_endpoint(clientFromString(reactor, SOCKS))
            tub.addConnectionHintHandler("tor", h)
        #else:
        #    h = tor.default_socks()
        #    tub.addConnectionHintHandler("tor", h)
    except ImportError:
        pass
    d = defer.succeed(None)
    d.addCallback(lambda _ign: tub.startService())
    d.addCallback(lambda _ign: tub.getReference(config.furl))
    d.addCallback(c.run, config.subOptions) # might provide tub here
    d.addBoth(lambda res: tub.stopService().addCallback(lambda _ign: res))
    return d
Beispiel #6
0
def do_remote_command(command, *args, **kwargs):
    client = AnyConsensoProcess()
    client.start()
    furl = client.furl()
    tub = Tub()
    tub.startService()

    def got_error(err):
        print "Error while calling command remotely", err
        reactor.stop()

    def got_result(res):
        print(res)

    def got_remote(remote):
        d = remote.callRemote(command, *args, **kwargs)
        d.addCallback(got_result)
        d.addCallback(lambda res: reactor.stop())
        d.addErrback(got_error)
        return d

    d = tub.getReference(furl)
    d.addCallback(got_remote)
    d.addErrback(got_error)
    reactor.run()
Beispiel #7
0
    def test_empty_location(self):
        # bug #129: a FURL with no location hints causes a synchronous
        # exception in Tub.getReference(), instead of an errback'ed Deferred.

        tubA = Tub()
        tubA.setServiceParent(self.s)
        tubB = Tub()
        tubB.setServiceParent(self.s)

        # This is a hack to get a FURL with empty location hints. The correct
        # way to make a Tub unreachable is to not call .setLocation() at all.
        tubB.setLocation("")
        r = Receiver(tubB)
        furl = tubB.registerReference(r)
        # the buggy behavior is that the following call raises an exception
        d = tubA.getReference(furl)
        # whereas it ought to return a Deferred
        self.assertTrue(isinstance(d, defer.Deferred))

        def _check(f):
            self.assertTrue(isinstance(f, failure.Failure), f)
            self.assertTrue(f.check(NoLocationHintsError), f)

        d.addBoth(_check)
        return d
Beispiel #8
0
 def testAuthenticated(self):
     url, portnum = self.makeServer()
     client = Tub()
     client.startService()
     self.services.append(client)
     d = client.getReference(url)
     return d
    def test_unreachable_client(self):
        # A "client-only" Tub has no location set. It should still be
        # possible to connect to objects in other (location-bearing) server
        # Tubs, and objects in the client Tub can still be sent to (and used
        # by) the server Tub.

        client_tub = Tub()
        client_tub.setServiceParent(self.s)
        server_tub = Tub()
        server_tub.setServiceParent(self.s)

        portnum = allocate_tcp_port()
        server_tub.listenOn("tcp:%d:interface=127.0.0.1" % portnum)
        server_tub.setLocation("tcp:127.0.0.1:%d" % portnum)
        s = Receiver() # no FURL, not directly reachable
        r = Receiver()
        furl = server_tub.registerReference(r)

        d = client_tub.getReference(furl)
        d.addCallback(lambda rref: rref.callRemote("call", s))
        d.addCallback(lambda res: self.failUnlessEqual(res, 1))
        d.addCallback(lambda _: self.failIfEqual(r.obj, None))
        def _inspect_obj(_):
            self.failUnlessEqual(r.obj.getSturdyRef().getURL(), None)
        d.addCallback(_inspect_obj)
        d.addCallback(lambda _: r.obj.callRemote("call", 2))
        d.addCallback(lambda _: self.failUnlessEqual(s.obj, 2))
        return d
Beispiel #10
0
 def test1(self):
     # Two FURLs pointing at the same Tub should share a connection. This
     # basically exercises TubRef.__cmp__ .
     furl1, furl2 = self.makeServers()
     client = Tub(certData_low)
     client.startService()
     self.services.append(client)
     d = client.getReference(furl1)
     rrefs = []
     d.addCallback(rrefs.append)
     d.addCallback(lambda _: client.getReference(furl2))
     d.addCallback(rrefs.append)
     def _check(_):
         self.failUnlessIdentical(rrefs[0].tracker.broker,
                                  rrefs[1].tracker.broker)
     d.addCallback(_check)
     return d
Beispiel #11
0
 def test1(self):
     # Two FURLs pointing at the same Tub should share a connection. This
     # basically exercises TubRef.__cmp__ .
     furl1, furl2 = self.makeServers()
     client = Tub(certData_low)
     client.startService()
     self.services.append(client)
     d = client.getReference(furl1)
     rrefs = []
     d.addCallback(rrefs.append)
     d.addCallback(lambda _: client.getReference(furl2))
     d.addCallback(rrefs.append)
     def _check(_):
         self.failUnlessIdentical(rrefs[0].tracker.broker,
                                  rrefs[1].tracker.broker)
     d.addCallback(_check)
     return d
Beispiel #12
0
 def test_queued_getref(self):
     t1 = Tub()
     d1 = t1.getReference(self.barry_url)
     d2 = t1.getReference(self.bill_url)
     def _check(res):
         ((barry_success, barry_rref),
          (bill_success, bill_rref)) = res
         self.assertTrue(barry_success)
         self.assertTrue(bill_success)
         self.assertTrue(isinstance(barry_rref, RemoteReference))
         self.assertTrue(isinstance(bill_rref, RemoteReference))
         self.assertFalse(barry_rref == bill_success)
     dl = defer.DeferredList([d1, d2])
     dl.addCallback(_check)
     self.services.append(t1)
     eventually(t1.startService)
     return dl
Beispiel #13
0
 def test_queued_getref(self):
     t1 = Tub()
     d1 = t1.getReference(self.barry_url)
     d2 = t1.getReference(self.bill_url)
     def _check(res):
         ((barry_success, barry_rref),
          (bill_success, bill_rref)) = res
         self.failUnless(barry_success)
         self.failUnless(bill_success)
         self.failUnless(isinstance(barry_rref, RemoteReference))
         self.failUnless(isinstance(bill_rref, RemoteReference))
         self.failIf(barry_rref == bill_success)
     dl = defer.DeferredList([d1, d2])
     dl.addCallback(_check)
     self.services.append(t1)
     eventually(t1.startService)
     return dl
Beispiel #14
0
def run_command(config):
    c = dispatch_table[config.subCommand]()
    tub = Tub()
    d = defer.succeed(None)
    d.addCallback(lambda _ign: tub.startService())
    d.addCallback(lambda _ign: tub.getReference(config.furl))
    d.addCallback(c.run, config.subOptions)  # might provide tub here
    d.addBoth(lambda res: tub.stopService().addCallback(lambda _ign: res))
    return d
Beispiel #15
0
def run_command(config):
    c = dispatch_table[config.subCommand]()
    tub = Tub()
    d = defer.succeed(None)
    d.addCallback(lambda _ign: tub.startService())
    d.addCallback(lambda _ign: tub.getReference(config.furl))
    d.addCallback(c.run, config.subOptions)  # might provide tub here
    d.addBoth(lambda res: tub.stopService().addCallback(lambda _ign: res))
    return d
Beispiel #16
0
 def testHalfAuthenticated2(self):
     if not crypto_available:
         raise unittest.SkipTest("crypto not available")
     url, portnum = self.makeServer(False)
     client = Tub()
     client.startService()
     self.services.append(client)
     d = client.getReference(url)
     return d
Beispiel #17
0
 def _testNoConnection_1(self, res, url):
     self.services.remove(self.tub)
     client = Tub()
     client.startService()
     self.services.append(client)
     d = client.getReference(url)
     d.addCallbacks(lambda res: self.fail("this is supposed to fail"),
                    self._testNoConnection_fail)
     return d
Beispiel #18
0
class FlappCommand(object):
    def __init__(self, furlfile):
        self.flappclient_args = ["-f", furlfile, "run-command"]
        options = ClientOptions()
        options.parseOptions(self.flappclient_args)
        self.furl = options.furl
        self.tub = Tub()
        self.rref = None
        self.d = defer.succeed(None)

    def start(self):
        self.d.addCallback(lambda ign: self.tub.startService())
        self.d.addCallback(lambda ign: self.tub.getReference(self.furl))

        done = defer.Deferred()
        def _got_rref(rref):
            self.rref = rref
            done.callback(None)
        self.d.addCallbacks(_got_rref, done.errback)
        return done

    def run(self, content, stdout, stderr, when_done, when_failed):
        assert self.rref is not None
        options = ClientOptions()
        options.stdout = stdout
        options.stderr = stderr
        options.parseOptions(self.flappclient_args)

        def stdio(proto):
            proto.dataReceived(content)
            proto.connectionLost("EOF")

        options.subOptions.stdio = stdio
        options.subOptions.stdout = stdout
        options.subOptions.stderr = stderr

        def _go(ign):
            print >>stdout, "Starting..."
            return RunCommand().run(self.rref, options.subOptions)
        def _done(rc):
            if rc == 0:
                when_done()
            else:
                print >>stdout, "Command failed with exit code %r." % (rc,)
                when_failed()
        def _error(f):
            print >>stdout, str(f)
            when_failed()
        def _recover(f):
            try:
                print >>stderr, str(f)
            except Exception:
                print >>stderr, "something weird"

        self.d.addCallback(_go)
        self.d.addCallbacks(_done, _error)
        self.d.addErrback(_recover)
Beispiel #19
0
 def _testNoConnection_1(self, res, url):
     self.services.remove(self.tub)
     client = Tub()
     client.startService()
     self.services.append(client)
     d = client.getReference(url)
     d.addCallbacks(lambda res: self.fail("this is supposed to fail"),
                    self._testNoConnection_fail)
     return d
Beispiel #20
0
class Server(unittest.TestCase, ShouldFailMixin):
    def setUp(self):
        self.s = service.MultiService()
        self.s.startService()
    def tearDown(self):
        return self.s.stopService()

    def run_cli(self, *args):
        argv = ["flappserver"] + list(args)
        d = defer.maybeDeferred(cli.run_flappserver, argv=argv, run_by_human=False)
        return d # fires with (rc,out,err)

    def test_run(self):
        basedir = "appserver/Server/run"
        os.makedirs(basedir)
        serverdir = os.path.join(basedir, "fl")
        incomingdir = os.path.join(basedir, "incoming")
        os.mkdir(incomingdir)

        self.tub = Tub()
        self.tub.setServiceParent(self.s)
        portnum = allocate_tcp_port()
        d = self.run_cli("create", "--location", "localhost:%d" % portnum,
                         "--port", "tcp:%d" % portnum,
                         serverdir)
        def _check((rc,out,err)):
            self.failUnlessEqual(rc, 0)
            self.failUnless(os.path.isdir(serverdir))
        d.addCallback(_check)
        d.addCallback(lambda ign: self.run_cli("add", serverdir,
                                               "upload-file", incomingdir))
        def _check_add((rc,out,err)):
            self.failUnlessEqual(rc, 0)
            lines = out.splitlines()
            self.failUnless(lines[1].startswith("FURL is pb://"))
            self.furl = lines[1].split()[-1]
        d.addCallback(_check_add)
        stdout = StringIO()
        def _start_server(ign):
            ap = server.AppServer(serverdir, stdout)
            ap.setServiceParent(self.s)
        d.addCallback(_start_server)
        # make sure the server can actually instantiate a service
        d.addCallback(lambda _ign: self.tub.getReference(self.furl))
        def _got_rref(rref):
            # great!
            pass
        d.addCallback(_got_rref)
        d.addCallback(lambda ign:
                      self.shouldFail(KeyError, "getReference(bogus)",
                                      "unable to find reference for name ",
                                      self.tub.getReference,
                                      self.furl+".bogus"))
        return d
Beispiel #21
0
 def testClientTimeout(self):
     portnum = self.makeNullServer()
     # lower the connection timeout to 2 seconds
     client = Tub(_test_options={'connect_timeout': 1})
     client.startService()
     self.services.append(client)
     url = "pb://[email protected]:%d/target" % portnum
     d = client.getReference(url)
     d.addCallbacks(lambda res: self.fail("hey! this is supposed to fail"),
                    lambda f: f.trap(tokens.NegotiationError))
     return d
Beispiel #22
0
 def testClientTimeout(self):
     portnum = self.makeNullServer()
     # lower the connection timeout to 2 seconds
     client = Tub(_test_options={'connect_timeout': 1})
     client.startService()
     self.services.append(client)
     url = "pb://[email protected]:%d/target" % portnum
     d = client.getReference(url)
     d.addCallbacks(lambda res: self.fail("hey! this is supposed to fail"),
                    lambda f: f.trap(tokens.NegotiationError))
     return d
Beispiel #23
0
 def testFuture3(self):
     # same as testFuture1, but it is the listening server that
     # understands [1,2]
     url, portnum = self.makeSpecificServer(certData_high, NegotiationVbig)
     client = Tub(certData=certData_low)
     client.startService()
     self.services.append(client)
     d = client.getReference(url)
     def _check_version(rref):
         ver = rref.tracker.broker._banana_decision_version
         self.failUnlessEqual(ver, MAX_HANDLED_VERSION)
     d.addCallback(_check_version)
     return d
Beispiel #24
0
 def testFuture3(self):
     # same as testFuture1, but it is the listening server that
     # understands [1,2]
     url, portnum = self.makeSpecificServer(certData_high, NegotiationVbig)
     client = Tub(certData=certData_low)
     client.startService()
     self.services.append(client)
     d = client.getReference(url)
     def _check_version(rref):
         ver = rref.tracker.broker._banana_decision_version
         self.failUnlessEqual(ver, MAX_HANDLED_VERSION)
     d.addCallback(_check_version)
     return d
Beispiel #25
0
class FlappCommand(object):
    def __init__(self, furlfile):
        self.flappclient_args = ["-f", furlfile, "run-command"]
        options = ClientOptions()
        options.parseOptions(self.flappclient_args)
        self.furl = options.furl
        self.tub = Tub()
        self.rref = None
        self.d = defer.succeed(None)

    def start(self):
        self.d.addCallback(lambda ign: self.tub.startService())
        self.d.addCallback(lambda ign: self.tub.getReference(self.furl))

        done = defer.Deferred()
        def _got_rref(rref):
            self.rref = rref
            done.callback(None)
        def _failed(f):
            done.errback(f)
            return f
        self.d.addCallbacks(_got_rref, _failed)
        return done

    def run(self, content, log):
        assert isinstance(content, bytes), (`content`, type(content))
        assert self.rref is not None
        options = ClientOptions()
        options.parseOptions(self.flappclient_args)

        def stdio(proto):
            # This value is being sent to the stdin of the flapp.
            proto.dataReceived(content)
            proto.connectionLost("EOF")

        options.subOptions.stdio = stdio

        # These are not used.
        options.subOptions.stdout = None
        options.subOptions.stderr = None

        print >>log, "Starting command."
        self.d = RunCommand().run(self.rref, options.subOptions)
        def _log_return_code(rc):
            print >>log, "Command completed with exit code %r" % (rc,)
        def _log_failure(f):
            print >>log, "Command failed with %r" % (f,)

        self.d.addCallbacks(_log_return_code, _log_failure)
        return self.d
Beispiel #26
0
class FlappCommand(object):
    def __init__(self, furl):
        self.flappclient_args = ["--furl", furl, "run-command"]
        options = ClientOptions()
        options.parseOptions(self.flappclient_args)
        self.furl = options.furl
        self.tub = Tub()
        self.rref = None
        self.d = defer.succeed(None)

    def start(self):
        self.d.addCallback(lambda ign: self.tub.startService())
        self.d.addCallback(lambda ign: self.tub.getReference(self.furl))

        done = defer.Deferred()
        def _got_rref(rref):
            self.rref = rref
            done.callback(None)
        def _failed(f):
            done.errback(f)
            return f
        self.d.addCallbacks(_got_rref, _failed)
        return done

    def run(self, content, log):
        assert isinstance(content, bytes), (`content`, type(content))
        assert self.rref is not None
        options = ClientOptions()
        options.parseOptions(self.flappclient_args)

        def stdio(proto):
            # This value is being sent to the stdin of the flapp.
            proto.dataReceived(content)
            proto.connectionLost("EOF")

        options.subOptions.stdio = stdio

        # These are not used.
        options.subOptions.stdout = None
        options.subOptions.stderr = None

        print >>log, "Starting command."
        self.d = RunCommand().run(self.rref, options.subOptions)
        def _log_return_code(rc):
            print >>log, "Command completed with exit code %r" % (rc,)
        def _log_failure(f):
            print >>log, "Command failed with %r" % (f,)

        self.d.addCallbacks(_log_return_code, _log_failure)
        return self.d
Beispiel #27
0
 def testVersusHTTPServerAuthenticated(self):
     portnum = self.makeHTTPServer()
     client = Tub()
     client.startService()
     self.services.append(client)
     url = "pb://%[email protected]:%d/target" % (tubid_low, portnum)
     d = client.getReference(url)
     d.addCallbacks(lambda res: self.fail("this is supposed to fail"),
                    lambda f: f.trap(BananaError))
     # the HTTP server needs a moment to notice that the connection has
     # gone away. Without this, trial flunks the test because of the
     # leftover HTTP server socket.
     d.addCallback(self.stall, 1)
     return d
Beispiel #28
0
 def testVersusHTTPServerAuthenticated(self):
     portnum = self.makeHTTPServer()
     client = Tub()
     client.startService()
     self.services.append(client)
     url = "pb://%[email protected]:%d/target" % (tubid_low, portnum)
     d = client.getReference(url)
     d.addCallbacks(lambda res: self.fail("this is supposed to fail"),
                    lambda f: f.trap(BananaError))
     # the HTTP server needs a moment to notice that the connection has
     # gone away. Without this, trial flunks the test because of the
     # leftover HTTP server socket.
     d.addCallback(self.stall, 1)
     return d
Beispiel #29
0
 def testTooFarInFuture4(self):
     # same as testTooFarInFuture2, but it is the listening server which
     # only understands [2]
     url, portnum = self.makeSpecificServer(certData_low,
                                            NegotiationVbigOnly)
     client = Tub(certData=certData_high)
     client.startService()
     self.services.append(client)
     d = client.getReference(url)
     def _oops_succeeded(rref):
         self.fail("hey! this is supposed to fail")
     def _check_failure(f):
         f.trap(tokens.NegotiationError, tokens.RemoteNegotiationError)
     d.addCallbacks(_oops_succeeded, _check_failure)
     return d
Beispiel #30
0
 def testFuture2(self):
     # same as before, but the connecting Tub will have the higher tubID,
     # and thus make the negotiation decision
     url, portnum = self.makeSpecificServer(certData_low)
     # the client
     client = Tub(certData=certData_high)
     client.negotiationClass = NegotiationVbig
     client.startService()
     self.services.append(client)
     d = client.getReference(url)
     def _check_version(rref):
         ver = rref.tracker.broker._banana_decision_version
         self.failUnlessEqual(ver, MAX_HANDLED_VERSION)
     d.addCallback(_check_version)
     return d
Beispiel #31
0
 def testTooFarInFuture2(self):
     # same as before, but the connecting Tub will have the higher tubID,
     # and thus make the negotiation decision
     url, portnum = self.makeSpecificServer(certData_low)
     client = Tub(certData=certData_high)
     client.negotiationClass = NegotiationVbigOnly
     client.startService()
     self.services.append(client)
     d = client.getReference(url)
     def _oops_succeeded(rref):
         self.fail("hey! this is supposed to fail")
     def _check_failure(f):
         f.trap(tokens.NegotiationError, tokens.RemoteNegotiationError)
     d.addCallbacks(_oops_succeeded, _check_failure)
     return d
Beispiel #32
0
 def testTooFarInFuture4(self):
     # same as testTooFarInFuture2, but it is the listening server which
     # only understands [2]
     url, portnum = self.makeSpecificServer(certData_low,
                                            NegotiationVbigOnly)
     client = Tub(certData=certData_high)
     client.startService()
     self.services.append(client)
     d = client.getReference(url)
     def _oops_succeeded(rref):
         self.fail("hey! this is supposed to fail")
     def _check_failure(f):
         f.trap(tokens.NegotiationError, tokens.RemoteNegotiationError)
     d.addCallbacks(_oops_succeeded, _check_failure)
     return d
Beispiel #33
0
 def testFuture2(self):
     # same as before, but the connecting Tub will have the higher tubID,
     # and thus make the negotiation decision
     url, portnum = self.makeSpecificServer(certData_low)
     # the client
     client = Tub(certData=certData_high)
     client.negotiationClass = NegotiationVbig
     client.startService()
     self.services.append(client)
     d = client.getReference(url)
     def _check_version(rref):
         ver = rref.tracker.broker._banana_decision_version
         self.failUnlessEqual(ver, MAX_HANDLED_VERSION)
     d.addCallback(_check_version)
     return d
Beispiel #34
0
 def testTooFarInFuture2(self):
     # same as before, but the connecting Tub will have the higher tubID,
     # and thus make the negotiation decision
     url, portnum = self.makeSpecificServer(certData_low)
     client = Tub(certData=certData_high)
     client.negotiationClass = NegotiationVbigOnly
     client.startService()
     self.services.append(client)
     d = client.getReference(url)
     def _oops_succeeded(rref):
         self.fail("hey! this is supposed to fail")
     def _check_failure(f):
         f.trap(tokens.NegotiationError, tokens.RemoteNegotiationError)
     d.addCallbacks(_oops_succeeded, _check_failure)
     return d
Beispiel #35
0
class TubFailures(ExamineFailuresMixin, ShouldFailMixin, unittest.TestCase):
    def setUp(self):
        self.s = service.MultiService()
        self.s.startService()
        self.target_tub = Tub()
        self.target_tub.setServiceParent(self.s)
        l = self.target_tub.listenOn("tcp:0:interface=127.0.0.1")
        self.target_tub.setLocation("127.0.0.1:%d" % l.getPortnum())
        self.source_tub = Tub()
        self.source_tub.setServiceParent(self.s)

    def tearDown(self):
        return self.s.stopService()

    def setupTarget(self, target):
        furl = self.target_tub.registerReference(target)
        d = self.source_tub.getReference(furl)
        return d


    def test_raise_not_exposed(self):
        self.source_tub.setOption("expose-remote-exception-types", False)
        d = self.setupTarget(TargetWithoutInterfaces())
        d.addCallback(lambda rr:
                      self.shouldFail(RemoteException, "one", None,
                                      rr.callRemote, "fail"))
        d.addCallback(self._examine_raise, True)
        return d

    def test_raise_yes_exposed(self):
        self.source_tub.setOption("expose-remote-exception-types", True)
        d = self.setupTarget(TargetWithoutInterfaces())
        d.addCallback(lambda rr:
                      self.shouldFail(ValueError, "one", None,
                                      rr.callRemote, "fail"))
        d.addCallback(self._examine_raise, False)
        return d

    def test_raise_default(self):
        # current default is to expose exceptions. This may change in the
        # future.
        d = self.setupTarget(TargetWithoutInterfaces())
        d.addCallback(lambda rr:
                      self.shouldFail(ValueError, "one", None,
                                      rr.callRemote, "fail"))
        d.addCallback(self._examine_raise, False)
        return d
Beispiel #36
0
    def testFuture1(self):
        # when a peer that understands version=[1] that connects to a peer
        # that understands version=[1,2], they should pick version=1

        # the listening Tub will have the higher tubID, and thus make the
        # negotiation decision
        url, portnum = self.makeSpecificServer(certData_high)
        # the client
        client = Tub(certData=certData_low)
        client.negotiationClass = NegotiationVbig
        client.startService()
        self.services.append(client)
        d = client.getReference(url)
        def _check_version(rref):
            ver = rref.tracker.broker._banana_decision_version
            self.failUnlessEqual(ver, MAX_HANDLED_VERSION)
        d.addCallback(_check_version)
        return d
Beispiel #37
0
    def testFuture1(self):
        # when a peer that understands version=[1] that connects to a peer
        # that understands version=[1,2], they should pick version=1

        # the listening Tub will have the higher tubID, and thus make the
        # negotiation decision
        url, portnum = self.makeSpecificServer(certData_high)
        # the client
        client = Tub(certData=certData_low)
        client.negotiationClass = NegotiationVbig
        client.startService()
        self.services.append(client)
        d = client.getReference(url)
        def _check_version(rref):
            ver = rref.tracker.broker._banana_decision_version
            self.failUnlessEqual(ver, MAX_HANDLED_VERSION)
        d.addCallback(_check_version)
        return d
Beispiel #38
0
    def testTooFarInFuture1(self):
        # when a peer that understands version=[1] that connects to a peer
        # that only understands version=[2], they should fail to negotiate

        # the listening Tub will have the higher tubID, and thus make the
        # negotiation decision
        url, portnum = self.makeSpecificServer(certData_high)
        # the client
        client = Tub(certData=certData_low)
        client.negotiationClass = NegotiationVbigOnly
        client.startService()
        self.services.append(client)
        d = client.getReference(url)
        def _oops_succeeded(rref):
            self.fail("hey! this is supposed to fail")
        def _check_failure(f):
            f.trap(tokens.NegotiationError, tokens.RemoteNegotiationError)
        d.addCallbacks(_oops_succeeded, _check_failure)
        return d
Beispiel #39
0
    def testTooFarInFuture1(self):
        # when a peer that understands version=[1] that connects to a peer
        # that only understands version=[2], they should fail to negotiate

        # the listening Tub will have the higher tubID, and thus make the
        # negotiation decision
        url, portnum = self.makeSpecificServer(certData_high)
        # the client
        client = Tub(certData=certData_low)
        client.negotiationClass = NegotiationVbigOnly
        client.startService()
        self.services.append(client)
        d = client.getReference(url)
        def _oops_succeeded(rref):
            self.fail("hey! this is supposed to fail")
        def _check_failure(f):
            f.trap(tokens.NegotiationError, tokens.RemoteNegotiationError)
        d.addCallbacks(_oops_succeeded, _check_failure)
        return d
Beispiel #40
0
class TubFailures(ExamineFailuresMixin, ShouldFailMixin, unittest.TestCase):
    def setUp(self):
        self.s = service.MultiService()
        self.s.startService()
        self.target_tub = Tub()
        self.target_tub.setServiceParent(self.s)
        l = self.target_tub.listenOn("tcp:0:interface=127.0.0.1")
        self.target_tub.setLocation("127.0.0.1:%d" % l.getPortnum())
        self.source_tub = Tub()
        self.source_tub.setServiceParent(self.s)

    def tearDown(self):
        return self.s.stopService()

    def setupTarget(self, target):
        furl = self.target_tub.registerReference(target)
        d = self.source_tub.getReference(furl)
        return d

    def test_raise_not_exposed(self):
        self.source_tub.setOption("expose-remote-exception-types", False)
        d = self.setupTarget(TargetWithoutInterfaces())
        d.addCallback(lambda rr: self.shouldFail(RemoteException, "one", None,
                                                 rr.callRemote, "fail"))
        d.addCallback(self._examine_raise, True)
        return d

    def test_raise_yes_exposed(self):
        self.source_tub.setOption("expose-remote-exception-types", True)
        d = self.setupTarget(TargetWithoutInterfaces())
        d.addCallback(lambda rr: self.shouldFail(ValueError, "one", None, rr.
                                                 callRemote, "fail"))
        d.addCallback(self._examine_raise, False)
        return d

    def test_raise_default(self):
        # current default is to expose exceptions. This may change in the
        # future.
        d = self.setupTarget(TargetWithoutInterfaces())
        d.addCallback(lambda rr: self.shouldFail(ValueError, "one", None, rr.
                                                 callRemote, "fail"))
        d.addCallback(self._examine_raise, False)
        return d
Beispiel #41
0
    def test_empty_location(self):
        # bug #129: a FURL with no location hints causes a synchronous
        # exception in Tub.getReference(), instead of an errback'ed Deferred.

        tubA = Tub()
        tubA.setServiceParent(self.s)
        tubB = Tub()
        tubB.setServiceParent(self.s)

        tubB.setLocation("") # this is how you say "unrouteable"
        r = Receiver(tubB)
        furl = tubB.registerReference(r)
        # the buggy behavior is that the following call raises an exception
        d = tubA.getReference(furl)
        # whereas it ought to return a Deferred
        self.failUnless(isinstance(d, defer.Deferred))
        def _check(f):
            self.failUnless(isinstance(f, failure.Failure), f)
            self.failUnless(f.check(NoLocationHintsError), f)
        d.addBoth(_check)
        return d
Beispiel #42
0
    def test_future(self):
        tubA = Tub()
        tubA.setServiceParent(self.s)
        tubB = Tub()
        tubB.setServiceParent(self.s)

        # "future:stuff" is interpreted as a "location hint format from the
        # future", which we're supposed to ignore, and are thus left with no
        # hints
        tubB.setLocation("future:stuff")
        r = Receiver(tubB)
        furl = tubB.registerReference(r)
        # the buggy behavior is that the following call raises an exception
        d = tubA.getReference(furl)
        # whereas it ought to return a Deferred
        self.assertTrue(isinstance(d, defer.Deferred))
        def _check(f):
            self.assertTrue(isinstance(f, failure.Failure), f)
            self.assertTrue(f.check(NoLocationHintsError), f)
        d.addBoth(_check)
        return d
Beispiel #43
0
    def test_future(self):
        tubA = Tub()
        tubA.setServiceParent(self.s)
        tubB = Tub()
        tubB.setServiceParent(self.s)

        # "future:stuff" is interpreted as a "location hint format from the
        # future", which we're supposed to ignore, and are thus left with no
        # hints
        tubB.setLocation("future:stuff")
        r = Receiver(tubB)
        furl = tubB.registerReference(r)
        # the buggy behavior is that the following call raises an exception
        d = tubA.getReference(furl)
        # whereas it ought to return a Deferred
        self.failUnless(isinstance(d, defer.Deferred))
        def _check(f):
            self.failUnless(isinstance(f, failure.Failure), f)
            self.failUnless(f.check(NoLocationHintsError), f)
        d.addBoth(_check)
        return d
Beispiel #44
0
 def testConnectAuthenticated(self):
     tub = Tub()
     self.startTub(tub)
     target = HelperTarget("bob")
     target.obj = "unset"
     url = tub.registerReference(target)
     # can we connect to a reference on our own Tub?
     d = tub.getReference(url)
     def _connected(ref):
         return ref.callRemote("set", 12)
     d.addCallback(_connected)
     def _check(res):
         self.failUnlessEqual(target.obj, 12)
     d.addCallback(_check)
     def _connect_again(res):
         target.obj = None
         return tub.getReference(url)
     d.addCallback(_connect_again)
     d.addCallback(_connected)
     d.addCallback(_check)
     return d
Beispiel #45
0
    def test_empty_location(self):
        # bug #129: a FURL with no location hints causes a synchronous
        # exception in Tub.getReference(), instead of an errback'ed Deferred.

        tubA = Tub()
        tubA.setServiceParent(self.s)
        tubB = Tub()
        tubB.setServiceParent(self.s)

        # This is a hack to get a FURL with empty location hints. The correct
        # way to make a Tub unreachable is to not call .setLocation() at all.
        tubB.setLocation("")
        r = Receiver(tubB)
        furl = tubB.registerReference(r)
        # the buggy behavior is that the following call raises an exception
        d = tubA.getReference(furl)
        # whereas it ought to return a Deferred
        self.failUnless(isinstance(d, defer.Deferred))
        def _check(f):
            self.failUnless(isinstance(f, failure.Failure), f)
            self.failUnless(f.check(NoLocationHintsError), f)
        d.addBoth(_check)
        return d
Beispiel #46
0
class CancelPendingDeliveries(unittest.TestCase, StallMixin):
    def tearDown(self):
        dl = [defer.succeed(None)]
        if self.tubA.running:
            dl.append(defer.maybeDeferred(self.tubA.stopService))
        if self.tubB.running:
            dl.append(defer.maybeDeferred(self.tubB.stopService))
        d = defer.DeferredList(dl)
        d.addCallback(flushEventualQueue)
        return d

    def test_cancel_pending_deliveries(self):
        # when a Tub is stopped, any deliveries that were pending should be
        # discarded. TubA sends remote_one+remote_two (and we hope they
        # arrive in the same chunk). TubB responds to remote_one by shutting
        # down. remote_two should be discarded. The bug was that remote_two
        # would cause an unhandled error on the TubB side.
        self.tubA = Tub()
        self.tubB = Tub()
        self.tubA.startService()
        self.tubB.startService()

        self.tubB.listenOn("tcp:0")
        d = self.tubB.setLocationAutomatically()
        r = Receiver(self.tubB)
        d.addCallback(lambda res: self.tubB.registerReference(r))
        d.addCallback(lambda furl: self.tubA.getReference(furl))
        def _go(rref):
            # we want these two to get sent and received in the same hunk
            rref.callRemoteOnly("one")
            rref.callRemoteOnly("two")
            return r.done_d
        d.addCallback(_go)
        # let remote_two do its log.err before we move on to the next test
        d.addCallback(self.stall, 1.0)
        return d
Beispiel #47
0
class CancelPendingDeliveries(unittest.TestCase, StallMixin):
    def tearDown(self):
        dl = [defer.succeed(None)]
        if self.tubA.running:
            dl.append(defer.maybeDeferred(self.tubA.stopService))
        if self.tubB.running:
            dl.append(defer.maybeDeferred(self.tubB.stopService))
        d = defer.DeferredList(dl)
        d.addCallback(flushEventualQueue)
        return d

    def test_cancel_pending_deliveries(self):
        # when a Tub is stopped, any deliveries that were pending should be
        # discarded. TubA sends remote_one+remote_two (and we hope they
        # arrive in the same chunk). TubB responds to remote_one by shutting
        # down. remote_two should be discarded. The bug was that remote_two
        # would cause an unhandled error on the TubB side.
        self.tubA = Tub()
        self.tubB = Tub()
        self.tubA.startService()
        self.tubB.startService()

        self.tubB.listenOn("tcp:0")
        d = self.tubB.setLocationAutomatically()
        r = Receiver(self.tubB)
        d.addCallback(lambda res: self.tubB.registerReference(r))
        d.addCallback(lambda furl: self.tubA.getReference(furl))
        def _go(rref):
            # we want these two to get sent and received in the same hunk
            rref.callRemoteOnly("one")
            rref.callRemoteOnly("two")
            return r.done_d
        d.addCallback(_go)
        # let remote_two do its log.err before we move on to the next test
        d.addCallback(self.stall, 1.0)
        return d
Beispiel #48
0
def printResult(number):
    print "the result is", number


def gotError(err):
    print "got an error:", err


def gotRemote(remote):
    o = Observer()
    d = remote.callRemote("addObserver", observer=o)
    d.addCallback(lambda res: remote.callRemote("push", num=2))
    d.addCallback(lambda res: remote.callRemote("push", num=3))
    d.addCallback(lambda res: remote.callRemote("add"))
    d.addCallback(lambda res: remote.callRemote("pop"))
    d.addCallback(printResult)
    d.addCallback(lambda res: remote.callRemote("removeObserver", observer=o))
    d.addErrback(gotError)
    d.addCallback(lambda res: reactor.stop())
    return d


url = sys.argv[1]
tub = Tub()
tub.startService()
d = tub.getReference(url)
d.addCallback(gotRemote)

reactor.run()
Beispiel #49
0
def getController(reactor, controllerFurl):
    t = Tub()
    #t.setOption("logLocalFailures", True)
    #t.setOption("logRemoteFailures", True)
    t.startService()
    return t.getReference(controllerFurl)
Beispiel #50
0
		reactor.stop()

	c += 1
	ctime = time.time()

	if c % 1000 == 0:
		print int(ctime*1000), c
	
	if ctime - starttime < 60.0:
		startcall(remote)
	else:
		print int(ctime*1000), c
		reactor.stop()

def gotReference(remote):
	startcall(remote)


tub = Tub()
tub.startService()
d = tub.getReference(sys.argv[1])
d.addCallbacks(gotReference, printexception)

c = 0
starttime = time.time()
ctime = time.time()
print int(ctime*1000), c

reactor.run()

Beispiel #51
0
 def connectClient(self, portnum):
     tub = Tub()
     tub.startService()
     self.services.append(tub)
     d = tub.getReference("pb://127.0.0.1:%d/hello" % portnum)
     return d
Beispiel #52
0
def gotError1(why):
    print "unable to get the RemoteReference:", why
    reactor.stop()

def gotError2(why):
    print "unable to invoke the remote method:", why
    reactor.stop()

def gotReference(remote):
    print "got a RemoteReference"
    print "asking it to add 1+2"
    d = remote.callRemote("add", a=1, b=2)
    d.addCallbacks(gotAnswer, gotError2)

def gotAnswer(answer):
    print "the answer is", answer
    reactor.stop()

if len(sys.argv) < 2:
    print "Usage: pb2client.py URL"
    sys.exit(1)
url = sys.argv[1]
tub = Tub()
tub.startService()
d = tub.getReference(url)
d.addCallbacks(gotReference, gotError1)

reactor.run()


Beispiel #53
0
class SystemFramework(pollmixin.PollMixin):
    numnodes = 7

    def __init__(self, basedir, mode):
        self.basedir = basedir = abspath_expanduser_unicode(str(basedir))
        if not (basedir + os.path.sep
                ).startswith(abspath_expanduser_unicode(u".") + os.path.sep):
            raise AssertionError("safety issue: basedir must be a subdir")
        self.testdir = testdir = os.path.join(basedir, "test")
        if os.path.exists(testdir):
            shutil.rmtree(testdir)
        fileutil.make_dirs(testdir)
        self.sparent = service.MultiService()
        self.sparent.startService()
        self.proc = None
        self.tub = Tub()
        self.tub.setOption("expose-remote-exception-types", False)
        self.tub.setServiceParent(self.sparent)
        self.mode = mode
        self.failed = False
        self.keepalive_file = None

    def run(self):
        framelog = os.path.join(self.basedir, "driver.log")
        log.startLogging(open(framelog, "a"), setStdout=False)
        log.msg("CHECK_MEMORY(mode=%s) STARTING" % self.mode)
        #logfile = open(os.path.join(self.testdir, "log"), "w")
        #flo = log.FileLogObserver(logfile)
        #log.startLoggingWithObserver(flo.emit, setStdout=False)
        d = fireEventually()
        d.addCallback(lambda res: self.setUp())
        d.addCallback(lambda res: self.record_initial_memusage())
        d.addCallback(lambda res: self.make_nodes())
        d.addCallback(lambda res: self.wait_for_client_connected())
        d.addCallback(lambda res: self.do_test())
        d.addBoth(self.tearDown)

        def _err(err):
            self.failed = err
            log.err(err)
            print(err)

        d.addErrback(_err)

        def _done(res):
            reactor.stop()
            return res

        d.addBoth(_done)
        reactor.run()
        if self.failed:
            # raiseException doesn't work for CopiedFailures
            self.failed.raiseException()

    def setUp(self):
        #print("STARTING")
        self.stats = {}
        self.statsfile = open(os.path.join(self.basedir, "stats.out"), "a")
        self.make_introducer()
        d = self.start_client()

        def _record_control_furl(control_furl):
            self.control_furl = control_furl
            #print("OBTAINING '%s'" % (control_furl,))
            return self.tub.getReference(self.control_furl)

        d.addCallback(_record_control_furl)

        def _record_control(control_rref):
            self.control_rref = control_rref

        d.addCallback(_record_control)

        def _ready(res):
            #print("CLIENT READY")
            pass

        d.addCallback(_ready)
        return d

    def record_initial_memusage(self):
        print()
        print("Client started (no connections yet)")
        d = self._print_usage()
        d.addCallback(self.stash_stats, "init")
        return d

    def wait_for_client_connected(self):
        print()
        print("Client connecting to other nodes..")
        return self.control_rref.callRemote("wait_for_client_connections",
                                            self.numnodes + 1)

    def tearDown(self, passthrough):
        # the client node will shut down in a few seconds
        #os.remove(os.path.join(self.clientdir, client.Client.EXIT_TRIGGER_FILE))
        log.msg("shutting down SystemTest services")
        if self.keepalive_file and os.path.exists(self.keepalive_file):
            age = time.time() - os.stat(self.keepalive_file)[stat.ST_MTIME]
            log.msg("keepalive file at shutdown was %ds old" % age)
        d = defer.succeed(None)
        if self.proc:
            d.addCallback(lambda res: self.kill_client())
        d.addCallback(lambda res: self.sparent.stopService())
        d.addCallback(lambda res: flushEventualQueue())

        def _close_statsfile(res):
            self.statsfile.close()

        d.addCallback(_close_statsfile)
        d.addCallback(lambda res: passthrough)
        return d

    def make_introducer(self):
        iv_basedir = os.path.join(self.testdir, "introducer")
        os.mkdir(iv_basedir)
        self.introducer = introducer.IntroducerNode(basedir=iv_basedir)
        self.introducer.setServiceParent(self)
        self.introducer_furl = self.introducer.introducer_url

    def make_nodes(self):
        self.nodes = []
        for i in range(self.numnodes):
            nodedir = os.path.join(self.testdir, "node%d" % i)
            os.mkdir(nodedir)
            f = open(os.path.join(nodedir, "tahoe.cfg"), "w")
            f.write("[client]\n"
                    "introducer.furl = %s\n"
                    "shares.happy = 1\n"
                    "[storage]\n" % (self.introducer_furl, ))
            # the only tests for which we want the internal nodes to actually
            # retain shares are the ones where somebody's going to download
            # them.
            if self.mode in ("download", "download-GET", "download-GET-slow"):
                # retain shares
                pass
            else:
                # for these tests, we tell the storage servers to pretend to
                # accept shares, but really just throw them out, since we're
                # only testing upload and not download.
                f.write("debug_discard = true\n")
            if self.mode in ("receive", ):
                # for this mode, the client-under-test gets all the shares,
                # so our internal nodes can refuse requests
                f.write("readonly = true\n")
            f.close()
            c = client.Client(basedir=nodedir)
            c.setServiceParent(self)
            self.nodes.append(c)
        # the peers will start running, eventually they will connect to each
        # other and the introducer

    def touch_keepalive(self):
        if os.path.exists(self.keepalive_file):
            age = time.time() - os.stat(self.keepalive_file)[stat.ST_MTIME]
            log.msg("touching keepalive file, was %ds old" % age)
        f = open(self.keepalive_file, "w")
        f.write("""\
If the node notices this file at startup, it will poll every 5 seconds and
terminate if the file is more than 10 seconds old, or if it has been deleted.
If the test harness has an internal failure and neglects to kill off the node
itself, this helps to avoid leaving processes lying around. The contents of
this file are ignored.
        """)
        f.close()

    def start_client(self):
        # this returns a Deferred that fires with the client's control.furl
        log.msg("MAKING CLIENT")
        # self.testdir is an absolute Unicode path
        clientdir = self.clientdir = os.path.join(self.testdir, u"client")
        clientdir_str = clientdir.encode(get_filesystem_encoding())
        quiet = StringIO()
        create_node.create_node({'basedir': clientdir}, out=quiet)
        log.msg("DONE MAKING CLIENT")
        # now replace tahoe.cfg
        # set webport=0 and then ask the node what port it picked.
        f = open(os.path.join(clientdir, "tahoe.cfg"), "w")
        f.write("[node]\n"
                "web.port = tcp:0:interface=127.0.0.1\n"
                "[client]\n"
                "introducer.furl = %s\n"
                "shares.happy = 1\n"
                "[storage]\n" % (self.introducer_furl, ))

        if self.mode in ("upload-self", "receive"):
            # accept and store shares, to trigger the memory consumption bugs
            pass
        else:
            # don't accept any shares
            f.write("readonly = true\n")
            ## also, if we do receive any shares, throw them away
            #f.write("debug_discard = true")
        if self.mode == "upload-self":
            pass
        f.close()
        self.keepalive_file = os.path.join(clientdir,
                                           client.Client.EXIT_TRIGGER_FILE)
        # now start updating the mtime.
        self.touch_keepalive()
        ts = internet.TimerService(1.0, self.touch_keepalive)
        ts.setServiceParent(self.sparent)

        pp = ClientWatcher()
        self.proc_done = pp.d = defer.Deferred()
        logfile = os.path.join(self.basedir, "client.log")
        tahoes = procutils.which("tahoe")
        if not tahoes:
            raise RuntimeError("unable to find a 'tahoe' executable")
        cmd = [tahoes[0], "run", ".", "-l", logfile]
        env = os.environ.copy()
        self.proc = reactor.spawnProcess(pp,
                                         cmd[0],
                                         cmd,
                                         env,
                                         path=clientdir_str)
        log.msg("CLIENT STARTED")

        # now we wait for the client to get started. we're looking for the
        # control.furl file to appear.
        furl_file = os.path.join(clientdir, "private", "control.furl")
        url_file = os.path.join(clientdir, "node.url")

        def _check():
            if pp.ended and pp.ended.value.status != 0:
                # the twistd process ends normally (with rc=0) if the child
                # is successfully launched. It ends abnormally (with rc!=0)
                # if the child cannot be launched.
                raise ChildDidNotStartError(
                    "process ended while waiting for startup")
            return os.path.exists(furl_file)

        d = self.poll(_check, 0.1)

        # once it exists, wait a moment before we read from it, just in case
        # it hasn't finished writing the whole thing. Ideally control.furl
        # would be created in some atomic fashion, or made non-readable until
        # it's ready, but I can't think of an easy way to do that, and I
        # think the chances that we'll observe a half-write are pretty low.
        def _stall(res):
            d2 = defer.Deferred()
            reactor.callLater(0.1, d2.callback, None)
            return d2

        d.addCallback(_stall)

        def _read(res):
            # read the node's URL
            self.webish_url = open(url_file, "r").read().strip()
            if self.webish_url[-1] == "/":
                # trim trailing slash, since the rest of the code wants it gone
                self.webish_url = self.webish_url[:-1]
            f = open(furl_file, "r")
            furl = f.read()
            return furl.strip()

        d.addCallback(_read)
        return d

    def kill_client(self):
        # returns a Deferred that fires when the process exits. This may only
        # be called once.
        try:
            self.proc.signalProcess("INT")
        except error.ProcessExitedAlready:
            pass
        return self.proc_done

    def create_data(self, name, size):
        filename = os.path.join(self.testdir, name + ".data")
        f = open(filename, "wb")
        block = "a" * 8192
        while size > 0:
            l = min(size, 8192)
            f.write(block[:l])
            size -= l
        return filename

    def stash_stats(self, stats, name):
        self.statsfile.write("%s %s: %d\n" %
                             (self.mode, name, stats['VmPeak']))
        self.statsfile.flush()
        self.stats[name] = stats['VmPeak']

    def POST(self, urlpath, **fields):
        url = self.webish_url + urlpath
        sepbase = "boogabooga"
        sep = "--" + sepbase
        form = []
        form.append(sep)
        form.append('Content-Disposition: form-data; name="_charset"')
        form.append('')
        form.append('UTF-8')
        form.append(sep)
        for name, value in fields.iteritems():
            if isinstance(value, tuple):
                filename, value = value
                form.append('Content-Disposition: form-data; name="%s"; '
                            'filename="%s"' % (name, filename))
            else:
                form.append('Content-Disposition: form-data; name="%s"' % name)
            form.append('')
            form.append(value)
            form.append(sep)
        form[-1] += "--"
        body = "\r\n".join(form) + "\r\n"
        headers = {
            "content-type": "multipart/form-data; boundary=%s" % sepbase,
        }
        return tw_client.getPage(url,
                                 method="POST",
                                 postdata=body,
                                 headers=headers,
                                 followRedirect=False)

    def GET_discard(self, urlpath, stall):
        url = self.webish_url + urlpath + "?filename=dummy-get.out"
        return discardPage(url, stall)

    def _print_usage(self, res=None):
        d = self.control_rref.callRemote("get_memory_usage")

        def _print(stats):
            print("VmSize: %9d  VmPeak: %9d" %
                  (stats["VmSize"], stats["VmPeak"]))
            return stats

        d.addCallback(_print)
        return d

    def _do_upload(self, res, size, files, uris):
        name = '%d' % size
        print()
        print("uploading %s" % name)
        if self.mode in ("upload", "upload-self"):
            d = self.control_rref.callRemote("upload_random_data_from_file",
                                             size,
                                             convergence="check-memory")
        elif self.mode == "upload-POST":
            data = "a" * size
            url = "/uri"
            d = self.POST(url, t="upload", file=("%d.data" % size, data))
        elif self.mode in ("receive", "download", "download-GET",
                           "download-GET-slow"):
            # mode=receive: upload the data from a local peer, so that the
            # client-under-test receives and stores the shares
            #
            # mode=download*: upload the data from a local peer, then have
            # the client-under-test download it.
            #
            # we need to wait until the uploading node has connected to all
            # peers, since the wait_for_client_connections() above doesn't
            # pay attention to our self.nodes[] and their connections.
            files[name] = self.create_data(name, size)
            u = self.nodes[0].getServiceNamed("uploader")
            d = self.nodes[0].debug_wait_for_client_connections(self.numnodes +
                                                                1)
            d.addCallback(lambda res: u.upload(
                upload.FileName(files[name], convergence="check-memory")))
            d.addCallback(lambda results: results.get_uri())
        else:
            raise ValueError("unknown mode=%s" % self.mode)

        def _complete(uri):
            uris[name] = uri
            print("uploaded %s" % name)

        d.addCallback(_complete)
        return d

    def _do_download(self, res, size, uris):
        if self.mode not in ("download", "download-GET", "download-GET-slow"):
            return
        name = '%d' % size
        print("downloading %s" % name)
        uri = uris[name]

        if self.mode == "download":
            d = self.control_rref.callRemote("download_to_tempfile_and_delete",
                                             uri)
        elif self.mode == "download-GET":
            url = "/uri/%s" % uri
            d = self.GET_discard(urllib.quote(url), stall=False)
        elif self.mode == "download-GET-slow":
            url = "/uri/%s" % uri
            d = self.GET_discard(urllib.quote(url), stall=True)

        def _complete(res):
            print("downloaded %s" % name)
            return res

        d.addCallback(_complete)
        return d

    def do_test(self):
        #print("CLIENT STARTED")
        #print("FURL", self.control_furl)
        #print("RREF", self.control_rref)
        #print()
        kB = 1000
        MB = 1000 * 1000
        files = {}
        uris = {}

        d = self._print_usage()
        d.addCallback(self.stash_stats, "0B")

        for i in range(10):
            d.addCallback(self._do_upload, 10 * kB + i, files, uris)
            d.addCallback(self._do_download, 10 * kB + i, uris)
            d.addCallback(self._print_usage)
        d.addCallback(self.stash_stats, "10kB")

        for i in range(3):
            d.addCallback(self._do_upload, 10 * MB + i, files, uris)
            d.addCallback(self._do_download, 10 * MB + i, uris)
            d.addCallback(self._print_usage)
        d.addCallback(self.stash_stats, "10MB")

        for i in range(1):
            d.addCallback(self._do_upload, 50 * MB + i, files, uris)
            d.addCallback(self._do_download, 50 * MB + i, uris)
            d.addCallback(self._print_usage)
        d.addCallback(self.stash_stats, "50MB")

        #for i in range(1):
        #    d.addCallback(self._do_upload, 100*MB+i, files, uris)
        #    d.addCallback(self._do_download, 100*MB+i, uris)
        #    d.addCallback(self._print_usage)
        #d.addCallback(self.stash_stats, "100MB")

        #d.addCallback(self.stall)
        def _done(res):
            print("FINISHING")

        d.addCallback(_done)
        return d

    def stall(self, res):
        d = defer.Deferred()
        reactor.callLater(5, d.callback, None)
        return d
Beispiel #54
0
class SystemFramework(pollmixin.PollMixin):
    numnodes = 7

    def __init__(self, basedir, mode):
        self.basedir = basedir = abspath_expanduser_unicode(unicode(basedir))
        if not (basedir + os.path.sep).startswith(abspath_expanduser_unicode(u".") + os.path.sep):
            raise AssertionError("safety issue: basedir must be a subdir")
        self.testdir = testdir = os.path.join(basedir, "test")
        if os.path.exists(testdir):
            shutil.rmtree(testdir)
        fileutil.make_dirs(testdir)
        self.sparent = service.MultiService()
        self.sparent.startService()
        self.proc = None
        self.tub = Tub()
        self.tub.setOption("expose-remote-exception-types", False)
        self.tub.setServiceParent(self.sparent)
        self.mode = mode
        self.failed = False
        self.keepalive_file = None

    def run(self):
        framelog = os.path.join(self.basedir, "driver.log")
        log.startLogging(open(framelog, "a"), setStdout=False)
        log.msg("CHECK_MEMORY(mode=%s) STARTING" % self.mode)
        #logfile = open(os.path.join(self.testdir, "log"), "w")
        #flo = log.FileLogObserver(logfile)
        #log.startLoggingWithObserver(flo.emit, setStdout=False)
        d = fireEventually()
        d.addCallback(lambda res: self.setUp())
        d.addCallback(lambda res: self.record_initial_memusage())
        d.addCallback(lambda res: self.make_nodes())
        d.addCallback(lambda res: self.wait_for_client_connected())
        d.addCallback(lambda res: self.do_test())
        d.addBoth(self.tearDown)
        def _err(err):
            self.failed = err
            log.err(err)
            print err
        d.addErrback(_err)
        def _done(res):
            reactor.stop()
            return res
        d.addBoth(_done)
        reactor.run()
        if self.failed:
            # raiseException doesn't work for CopiedFailures
            self.failed.raiseException()

    def setUp(self):
        #print "STARTING"
        self.stats = {}
        self.statsfile = open(os.path.join(self.basedir, "stats.out"), "a")
        d = self.make_introducer()
        def _more(res):
            return self.start_client()
        d.addCallback(_more)
        def _record_control_furl(control_furl):
            self.control_furl = control_furl
            #print "OBTAINING '%s'" % (control_furl,)
            return self.tub.getReference(self.control_furl)
        d.addCallback(_record_control_furl)
        def _record_control(control_rref):
            self.control_rref = control_rref
        d.addCallback(_record_control)
        def _ready(res):
            #print "CLIENT READY"
            pass
        d.addCallback(_ready)
        return d

    def record_initial_memusage(self):
        print
        print "Client started (no connections yet)"
        d = self._print_usage()
        d.addCallback(self.stash_stats, "init")
        return d

    def wait_for_client_connected(self):
        print
        print "Client connecting to other nodes.."
        return self.control_rref.callRemote("wait_for_client_connections",
                                            self.numnodes+1)

    def tearDown(self, passthrough):
        # the client node will shut down in a few seconds
        #os.remove(os.path.join(self.clientdir, "suicide_prevention_hotline"))
        log.msg("shutting down SystemTest services")
        if self.keepalive_file and os.path.exists(self.keepalive_file):
            age = time.time() - os.stat(self.keepalive_file)[stat.ST_MTIME]
            log.msg("keepalive file at shutdown was %ds old" % age)
        d = defer.succeed(None)
        if self.proc:
            d.addCallback(lambda res: self.kill_client())
        d.addCallback(lambda res: self.sparent.stopService())
        d.addCallback(lambda res: flushEventualQueue())
        def _close_statsfile(res):
            self.statsfile.close()
        d.addCallback(_close_statsfile)
        d.addCallback(lambda res: passthrough)
        return d

    def add_service(self, s):
        s.setServiceParent(self.sparent)
        return s

    def make_introducer(self):
        iv_basedir = os.path.join(self.testdir, "introducer")
        os.mkdir(iv_basedir)
        iv = introducer.IntroducerNode(basedir=iv_basedir)
        self.introducer = self.add_service(iv)
        d = self.introducer.when_tub_ready()
        def _introducer_ready(res):
            q = self.introducer
            self.introducer_furl = q.introducer_url
        d.addCallback(_introducer_ready)
        return d

    def make_nodes(self):
        self.nodes = []
        for i in range(self.numnodes):
            nodedir = os.path.join(self.testdir, "node%d" % i)
            os.mkdir(nodedir)
            f = open(os.path.join(nodedir, "tahoe.cfg"), "w")
            f.write("[client]\n"
                    "introducer.furl = %s\n"
                    "shares.happy = 1\n"
                    "[storage]\n"
                    % (self.introducer_furl,))
            # the only tests for which we want the internal nodes to actually
            # retain shares are the ones where somebody's going to download
            # them.
            if self.mode in ("download", "download-GET", "download-GET-slow"):
                # retain shares
                pass
            else:
                # for these tests, we tell the storage servers to pretend to
                # accept shares, but really just throw them out, since we're
                # only testing upload and not download.
                f.write("debug_discard = true\n")
            if self.mode in ("receive",):
                # for this mode, the client-under-test gets all the shares,
                # so our internal nodes can refuse requests
                f.write("readonly = true\n")
            f.close()
            c = self.add_service(client.Client(basedir=nodedir))
            self.nodes.append(c)
        # the peers will start running, eventually they will connect to each
        # other and the introducer

    def touch_keepalive(self):
        if os.path.exists(self.keepalive_file):
            age = time.time() - os.stat(self.keepalive_file)[stat.ST_MTIME]
            log.msg("touching keepalive file, was %ds old" % age)
        f = open(self.keepalive_file, "w")
        f.write("""\
If the node notices this file at startup, it will poll every 5 seconds and
terminate if the file is more than 10 seconds old, or if it has been deleted.
If the test harness has an internal failure and neglects to kill off the node
itself, this helps to avoid leaving processes lying around. The contents of
this file are ignored.
        """)
        f.close()

    def start_client(self):
        # this returns a Deferred that fires with the client's control.furl
        log.msg("MAKING CLIENT")
        # self.testdir is an absolute Unicode path
        clientdir = self.clientdir = os.path.join(self.testdir, u"client")
        clientdir_str = clientdir.encode(get_filesystem_encoding())
        quiet = StringIO()
        create_node.create_node({'basedir': clientdir}, out=quiet)
        log.msg("DONE MAKING CLIENT")
        # now replace tahoe.cfg
        # set webport=0 and then ask the node what port it picked.
        f = open(os.path.join(clientdir, "tahoe.cfg"), "w")
        f.write("[node]\n"
                "web.port = tcp:0:interface=127.0.0.1\n"
                "[client]\n"
                "introducer.furl = %s\n"
                "shares.happy = 1\n"
                "[storage]\n"
                % (self.introducer_furl,))

        if self.mode in ("upload-self", "receive"):
            # accept and store shares, to trigger the memory consumption bugs
            pass
        else:
            # don't accept any shares
            f.write("readonly = true\n")
            ## also, if we do receive any shares, throw them away
            #f.write("debug_discard = true")
        if self.mode == "upload-self":
            pass
        f.close()
        self.keepalive_file = os.path.join(clientdir,
                                           "suicide_prevention_hotline")
        # now start updating the mtime.
        self.touch_keepalive()
        ts = internet.TimerService(1.0, self.touch_keepalive)
        ts.setServiceParent(self.sparent)

        pp = ClientWatcher()
        self.proc_done = pp.d = defer.Deferred()
        logfile = os.path.join(self.basedir, "client.log")
        cmd = ["twistd", "-n", "-y", "tahoe-client.tac", "-l", logfile]
        env = os.environ.copy()
        self.proc = reactor.spawnProcess(pp, cmd[0], cmd, env, path=clientdir_str)
        log.msg("CLIENT STARTED")

        # now we wait for the client to get started. we're looking for the
        # control.furl file to appear.
        furl_file = os.path.join(clientdir, "private", "control.furl")
        url_file = os.path.join(clientdir, "node.url")
        def _check():
            if pp.ended and pp.ended.value.status != 0:
                # the twistd process ends normally (with rc=0) if the child
                # is successfully launched. It ends abnormally (with rc!=0)
                # if the child cannot be launched.
                raise ChildDidNotStartError("process ended while waiting for startup")
            return os.path.exists(furl_file)
        d = self.poll(_check, 0.1)
        # once it exists, wait a moment before we read from it, just in case
        # it hasn't finished writing the whole thing. Ideally control.furl
        # would be created in some atomic fashion, or made non-readable until
        # it's ready, but I can't think of an easy way to do that, and I
        # think the chances that we'll observe a half-write are pretty low.
        def _stall(res):
            d2 = defer.Deferred()
            reactor.callLater(0.1, d2.callback, None)
            return d2
        d.addCallback(_stall)
        def _read(res):
            # read the node's URL
            self.webish_url = open(url_file, "r").read().strip()
            if self.webish_url[-1] == "/":
                # trim trailing slash, since the rest of the code wants it gone
                self.webish_url = self.webish_url[:-1]
            f = open(furl_file, "r")
            furl = f.read()
            return furl.strip()
        d.addCallback(_read)
        return d


    def kill_client(self):
        # returns a Deferred that fires when the process exits. This may only
        # be called once.
        try:
            self.proc.signalProcess("INT")
        except error.ProcessExitedAlready:
            pass
        return self.proc_done


    def create_data(self, name, size):
        filename = os.path.join(self.testdir, name + ".data")
        f = open(filename, "wb")
        block = "a" * 8192
        while size > 0:
            l = min(size, 8192)
            f.write(block[:l])
            size -= l
        return filename

    def stash_stats(self, stats, name):
        self.statsfile.write("%s %s: %d\n" % (self.mode, name, stats['VmPeak']))
        self.statsfile.flush()
        self.stats[name] = stats['VmPeak']

    def POST(self, urlpath, **fields):
        url = self.webish_url + urlpath
        sepbase = "boogabooga"
        sep = "--" + sepbase
        form = []
        form.append(sep)
        form.append('Content-Disposition: form-data; name="_charset"')
        form.append('')
        form.append('UTF-8')
        form.append(sep)
        for name, value in fields.iteritems():
            if isinstance(value, tuple):
                filename, value = value
                form.append('Content-Disposition: form-data; name="%s"; '
                            'filename="%s"' % (name, filename))
            else:
                form.append('Content-Disposition: form-data; name="%s"' % name)
            form.append('')
            form.append(value)
            form.append(sep)
        form[-1] += "--"
        body = "\r\n".join(form) + "\r\n"
        headers = {"content-type": "multipart/form-data; boundary=%s" % sepbase,
                   }
        return tw_client.getPage(url, method="POST", postdata=body,
                                 headers=headers, followRedirect=False)

    def GET_discard(self, urlpath, stall):
        url = self.webish_url + urlpath + "?filename=dummy-get.out"
        return discardPage(url, stall)

    def _print_usage(self, res=None):
        d = self.control_rref.callRemote("get_memory_usage")
        def _print(stats):
            print "VmSize: %9d  VmPeak: %9d" % (stats["VmSize"],
                                                stats["VmPeak"])
            return stats
        d.addCallback(_print)
        return d

    def _do_upload(self, res, size, files, uris):
        name = '%d' % size
        print
        print "uploading %s" % name
        if self.mode in ("upload", "upload-self"):
            files[name] = self.create_data(name, size)
            d = self.control_rref.callRemote("upload_from_file_to_uri",
                                             files[name].encode("utf-8"),
                                             convergence="check-memory")
            def _done(uri):
                os.remove(files[name])
                del files[name]
                return uri
            d.addCallback(_done)
        elif self.mode == "upload-POST":
            data = "a" * size
            url = "/uri"
            d = self.POST(url, t="upload", file=("%d.data" % size, data))
        elif self.mode in ("receive",
                           "download", "download-GET", "download-GET-slow"):
            # mode=receive: upload the data from a local peer, so that the
            # client-under-test receives and stores the shares
            #
            # mode=download*: upload the data from a local peer, then have
            # the client-under-test download it.
            #
            # we need to wait until the uploading node has connected to all
            # peers, since the wait_for_client_connections() above doesn't
            # pay attention to our self.nodes[] and their connections.
            files[name] = self.create_data(name, size)
            u = self.nodes[0].getServiceNamed("uploader")
            d = self.nodes[0].debug_wait_for_client_connections(self.numnodes+1)
            d.addCallback(lambda res:
                          u.upload(upload.FileName(files[name],
                                                   convergence="check-memory")))
            d.addCallback(lambda results: results.uri)
        else:
            raise ValueError("unknown mode=%s" % self.mode)
        def _complete(uri):
            uris[name] = uri
            print "uploaded %s" % name
        d.addCallback(_complete)
        return d

    def _do_download(self, res, size, uris):
        if self.mode not in ("download", "download-GET", "download-GET-slow"):
            return
        name = '%d' % size
        print "downloading %s" % name
        uri = uris[name]

        if self.mode == "download":
            d = self.control_rref.callRemote("download_from_uri_to_file",
                                             uri, "dummy.out")
        elif self.mode == "download-GET":
            url = "/uri/%s" % uri
            d = self.GET_discard(urllib.quote(url), stall=False)
        elif self.mode == "download-GET-slow":
            url = "/uri/%s" % uri
            d = self.GET_discard(urllib.quote(url), stall=True)

        def _complete(res):
            print "downloaded %s" % name
            return res
        d.addCallback(_complete)
        return d

    def do_test(self):
        #print "CLIENT STARTED"
        #print "FURL", self.control_furl
        #print "RREF", self.control_rref
        #print
        kB = 1000; MB = 1000*1000
        files = {}
        uris = {}

        d = self._print_usage()
        d.addCallback(self.stash_stats, "0B")

        for i in range(10):
            d.addCallback(self._do_upload, 10*kB+i, files, uris)
            d.addCallback(self._do_download, 10*kB+i, uris)
            d.addCallback(self._print_usage)
        d.addCallback(self.stash_stats, "10kB")

        for i in range(3):
            d.addCallback(self._do_upload, 10*MB+i, files, uris)
            d.addCallback(self._do_download, 10*MB+i, uris)
            d.addCallback(self._print_usage)
        d.addCallback(self.stash_stats, "10MB")

        for i in range(1):
            d.addCallback(self._do_upload, 50*MB+i, files, uris)
            d.addCallback(self._do_download, 50*MB+i, uris)
            d.addCallback(self._print_usage)
        d.addCallback(self.stash_stats, "50MB")

        #for i in range(1):
        #    d.addCallback(self._do_upload, 100*MB+i, files, uris)
        #    d.addCallback(self._do_download, 100*MB+i, uris)
        #    d.addCallback(self._print_usage)
        #d.addCallback(self.stash_stats, "100MB")

        #d.addCallback(self.stall)
        def _done(res):
            print "FINISHING"
        d.addCallback(_done)
        return d

    def stall(self, res):
        d = defer.Deferred()
        reactor.callLater(5, d.callback, None)
        return d
Beispiel #55
0
 def connectClient(self, portnum):
     tub = Tub()
     tub.startService()
     self.services.append(tub)
     d = tub.getReference("pb://127.0.0.1:%d/hello" % portnum)
     return d