class LogTail: def __init__(self, options): self.options = options def run(self, target_furl): target_tubid = SturdyRef(target_furl).getTubRef().getTubID() d = fireEventually(target_furl) d.addCallback(self.start, target_tubid) d.addErrback(self._error) print("starting..") reactor.run() def _error(self, f): print("ERROR", f) reactor.stop() def start(self, target_furl, target_tubid): print("Connecting..") self._tub = Tub() self._tub.startService() self._tub.connectTo(target_furl, self._got_logpublisher, target_tubid) def _got_logpublisher(self, publisher, target_tubid): d = publisher.callRemote("get_pid") def _announce(pid_or_failure): if isinstance(pid_or_failure, int): print("Connected (to pid %d)" % pid_or_failure) return pid_or_failure else: # the logport is probably foolscap-0.2.8 or earlier and # doesn't offer get_pid() print("Connected (unable to get pid)") return None d.addBoth(_announce) publisher.notifyOnDisconnect(self._lost_logpublisher) lp = LogPrinter(self.options, target_tubid) def _ask_for_versions(pid): d = publisher.callRemote("get_versions") d.addCallback(lp.got_versions, pid) return d d.addCallback(_ask_for_versions) catch_up = bool(self.options["catch-up"]) if catch_up: d.addCallback( lambda res: publisher.callRemote("subscribe_to_all", lp, True)) else: # provide compatibility with foolscap-0.2.4 and earlier, which # didn't accept a catchup= argument d.addCallback( lambda res: publisher.callRemote("subscribe_to_all", lp)) d.addErrback(self._error) return d def _lost_logpublisher(publisher): print("Disconnected")
class LogTail: def __init__(self, options): self.options = options def run(self, target_furl): target_tubid = SturdyRef(target_furl).getTubRef().getTubID() d = fireEventually(target_furl) d.addCallback(self.start, target_tubid) d.addErrback(self._error) print "starting.." reactor.run() def _error(self, f): print "ERROR", f reactor.stop() def start(self, target_furl, target_tubid): print "Connecting.." self._tub = Tub() self._tub.startService() self._tub.connectTo(target_furl, self._got_logpublisher, target_tubid) def _got_logpublisher(self, publisher, target_tubid): d = publisher.callRemote("get_pid") def _announce(pid_or_failure): if isinstance(pid_or_failure, int): print "Connected (to pid %d)" % pid_or_failure return pid_or_failure else: # the logport is probably foolscap-0.2.8 or earlier and # doesn't offer get_pid() print "Connected (unable to get pid)" return None d.addBoth(_announce) publisher.notifyOnDisconnect(self._lost_logpublisher) lp = LogPrinter(self.options, target_tubid) def _ask_for_versions(pid): d = publisher.callRemote("get_versions") d.addCallback(lp.got_versions, pid) return d d.addCallback(_ask_for_versions) catch_up = bool(self.options["catch-up"]) if catch_up: d.addCallback(lambda res: publisher.callRemote("subscribe_to_all", lp, True)) else: # provide compatibility with foolscap-0.2.4 and earlier, which # didn't accept a catchup= argument d.addCallback(lambda res: publisher.callRemote("subscribe_to_all", lp)) d.addErrback(self._error) return d def _lost_logpublisher(publisher): print "Disconnected"
def test_queued_reconnector(self): t1 = Tub() bill_connections = [] barry_connections = [] t1.connectTo(self.bill_url, bill_connections.append) t1.connectTo(self.barry_url, barry_connections.append) def _check(): if len(bill_connections) >= 1 and len(barry_connections) >= 1: return True return False d = self.poll(_check) def _validate(res): self.failUnless(isinstance(bill_connections[0], RemoteReference)) self.failUnless(isinstance(barry_connections[0], RemoteReference)) self.failIf(bill_connections[0] == barry_connections[0]) d.addCallback(_validate) self.services.append(t1) eventually(t1.startService) return d
def test_queued_reconnector(self): t1 = Tub() bill_connections = [] barry_connections = [] t1.connectTo(self.bill_url, bill_connections.append) t1.connectTo(self.barry_url, barry_connections.append) def _check(): if len(bill_connections) >= 1 and len(barry_connections) >= 1: return True return False d = self.poll(_check) def _validate(res): self.assertTrue(isinstance(bill_connections[0], RemoteReference)) self.assertTrue(isinstance(barry_connections[0], RemoteReference)) self.assertFalse(bill_connections[0] == barry_connections[0]) d.addCallback(_validate) self.services.append(t1) eventually(t1.startService) return d
class Failed(unittest.TestCase): def setUp(self): self.services = [] def tearDown(self): d = defer.DeferredList([s.stopService() for s in self.services]) d.addCallback(flushEventualQueue) return d @defer.inlineCallbacks def test_bad_hints(self): self.tubA = Tub() self.tubA.startService() self.services.append(self.tubA) self.tubB = Tub() self.tubB.startService() self.services.append(self.tubB) portnum = allocate_tcp_port() self.tubB.listenOn("tcp:%d:interface=127.0.0.1" % portnum) bad1 = "no-colon" bad2 = "unknown:foo" bad3 = "tcp:300.300.300.300:333" self.tubB.setLocation(bad1, bad2, bad3) target = HelperTarget("bob") url = self.tubB.registerReference(target) rc = self.tubA.connectTo(url, None) ri = rc.getReconnectionInfo() self.assertEqual(ri.state, "connecting") d = defer.Deferred() reactor.callLater(1.0, d.callback, None) yield d # now look at the details ri = rc.getReconnectionInfo() self.assertEqual(ri.state, "waiting") ci = ri.connectionInfo self.assertEqual(ci.connected, False) self.assertEqual(ci.winningHint, None) s = ci.connectorStatuses self.assertEqual(set(s.keys()), set([bad1, bad2, bad3])) self.assertEqual(s[bad1], "bad hint: no colon") self.assertEqual(s[bad2], "bad hint: no handler registered") self.assertIn("DNS lookup failed", s[bad3]) ch = ci.connectionHandlers self.assertEqual(ch, {bad2: None, bad3: "tcp"})
class NativeStorageServer(service.MultiService): """I hold information about a storage server that we want to connect to. If we are connected, I hold the RemoteReference, their host address, and the their version information. I remember information about when we were last connected too, even if we aren't currently connected. @ivar last_connect_time: when we last established a connection @ivar last_loss_time: when we last lost a connection @ivar version: the server's versiondict, from the most recent announcement @ivar nickname: the server's self-reported nickname (unicode), same @ivar rref: the RemoteReference, if connected, otherwise None @ivar remote_host: the IAddress, if connected, otherwise None """ implements(IServer) VERSION_DEFAULTS = { "http://allmydata.org/tahoe/protocols/storage/v1" : { "maximum-immutable-share-size": 2**32 - 1, "maximum-mutable-share-size": 2*1000*1000*1000, # maximum prior to v1.9.2 "tolerates-immutable-read-overrun": False, "delete-mutable-shares-with-zero-length-writev": False, "available-space": None, }, "application-version": "unknown: no get_version()", } def __init__(self, key_s, ann, tub_options={}): service.MultiService.__init__(self) self.key_s = key_s self.announcement = ann self._tub_options = tub_options assert "anonymous-storage-FURL" in ann, ann furl = str(ann["anonymous-storage-FURL"]) m = re.match(r'pb://(\w+)@', furl) assert m, furl tubid_s = m.group(1).lower() self._tubid = base32.a2b(tubid_s) assert "permutation-seed-base32" in ann, ann ps = base32.a2b(str(ann["permutation-seed-base32"])) self._permutation_seed = ps if key_s: self._long_description = key_s if key_s.startswith("v0-"): # remove v0- prefix from abbreviated name self._short_description = key_s[3:3+8] else: self._short_description = key_s[:8] else: self._long_description = tubid_s self._short_description = tubid_s[:6] self.last_connect_time = None self.last_loss_time = None self.remote_host = None self.rref = None self._is_connected = False self._reconnector = None self._trigger_cb = None self._on_status_changed = ObserverList() def on_status_changed(self, status_changed): """ :param status_changed: a callable taking a single arg (the NativeStorageServer) that is notified when we become connected """ return self._on_status_changed.subscribe(status_changed) # Special methods used by copy.copy() and copy.deepcopy(). When those are # used in allmydata.immutable.filenode to copy CheckResults during # repair, we want it to treat the IServer instances as singletons, and # not attempt to duplicate them.. def __copy__(self): return self def __deepcopy__(self, memodict): return self def __repr__(self): return "<NativeStorageServer for %s>" % self.get_name() def get_serverid(self): return self._tubid # XXX replace with self.key_s def get_permutation_seed(self): return self._permutation_seed def get_version(self): if self.rref: return self.rref.version return None def get_name(self): # keep methodname short # TODO: decide who adds [] in the short description. It should # probably be the output side, not here. return self._short_description def get_longname(self): return self._long_description def get_lease_seed(self): return self._tubid def get_foolscap_write_enabler_seed(self): return self._tubid def get_nickname(self): return self.announcement["nickname"] def get_announcement(self): return self.announcement def get_remote_host(self): return self.remote_host def is_connected(self): return self._is_connected def get_last_connect_time(self): return self.last_connect_time def get_last_loss_time(self): return self.last_loss_time def get_last_received_data_time(self): if self.rref is None: return None else: return self.rref.getDataLastReceivedAt() def get_available_space(self): version = self.get_version() if version is None: return None protocol_v1_version = version.get('http://allmydata.org/tahoe/protocols/storage/v1', {}) available_space = protocol_v1_version.get('available-space') if available_space is None: available_space = protocol_v1_version.get('maximum-immutable-share-size', None) return available_space def start_connecting(self, trigger_cb): self._tub = Tub() for (name, value) in self._tub_options.items(): self._tub.setOption(name, value) self._tub.setServiceParent(self) furl = str(self.announcement["anonymous-storage-FURL"]) self._trigger_cb = trigger_cb self._reconnector = self._tub.connectTo(furl, self._got_connection) def _got_connection(self, rref): lp = log.msg(format="got connection to %(name)s, getting versions", name=self.get_name(), facility="tahoe.storage_broker", umid="coUECQ") if self._trigger_cb: eventually(self._trigger_cb) default = self.VERSION_DEFAULTS d = add_version_to_remote_reference(rref, default) d.addCallback(self._got_versioned_service, lp) d.addCallback(lambda ign: self._on_status_changed.notify(self)) d.addErrback(log.err, format="storageclient._got_connection", name=self.get_name(), umid="Sdq3pg") def _got_versioned_service(self, rref, lp): log.msg(format="%(name)s provided version info %(version)s", name=self.get_name(), version=rref.version, facility="tahoe.storage_broker", umid="SWmJYg", level=log.NOISY, parent=lp) self.last_connect_time = time.time() self.remote_host = rref.getPeer() self.rref = rref self._is_connected = True rref.notifyOnDisconnect(self._lost) def get_rref(self): return self.rref def _lost(self): log.msg(format="lost connection to %(name)s", name=self.get_name(), facility="tahoe.storage_broker", umid="zbRllw") self.last_loss_time = time.time() # self.rref is now stale: all callRemote()s will get a # DeadReferenceError. We leave the stale reference in place so that # uploader/downloader code (which received this IServer through # get_connected_servers() or get_servers_for_psi()) can continue to # use s.get_rref().callRemote() and not worry about it being None. self._is_connected = False self.remote_host = None def stop_connecting(self): # used when this descriptor has been superceded by another self._reconnector.stopConnecting() def try_to_connect(self): # used when the broker wants us to hurry up self._reconnector.reset()
class NativeStorageServer(service.MultiService): """I hold information about a storage server that we want to connect to. If we are connected, I hold the RemoteReference, their host address, and the their version information. I remember information about when we were last connected too, even if we aren't currently connected. @ivar last_connect_time: when we last established a connection @ivar last_loss_time: when we last lost a connection @ivar version: the server's versiondict, from the most recent announcement @ivar nickname: the server's self-reported nickname (unicode), same @ivar rref: the RemoteReference, if connected, otherwise None @ivar remote_host: the IAddress, if connected, otherwise None """ implements(IServer) VERSION_DEFAULTS = { "http://allmydata.org/tahoe/protocols/storage/v1": { "maximum-immutable-share-size": 2**32 - 1, "maximum-mutable-share-size": 2 * 1000 * 1000 * 1000, # maximum prior to v1.9.2 "tolerates-immutable-read-overrun": False, "delete-mutable-shares-with-zero-length-writev": False, "available-space": None, }, "application-version": "unknown: no get_version()", } def __init__(self, key_s, ann, tub_options={}, tub_handlers={}): service.MultiService.__init__(self) self.key_s = key_s self.announcement = ann self._tub_options = tub_options self._tub_handlers = tub_handlers assert "anonymous-storage-FURL" in ann, ann furl = str(ann["anonymous-storage-FURL"]) m = re.match(r'pb://(\w+)@', furl) assert m, furl tubid_s = m.group(1).lower() self._tubid = base32.a2b(tubid_s) assert "permutation-seed-base32" in ann, ann ps = base32.a2b(str(ann["permutation-seed-base32"])) self._permutation_seed = ps if key_s: self._long_description = key_s if key_s.startswith("v0-"): # remove v0- prefix from abbreviated name self._short_description = key_s[3:3 + 8] else: self._short_description = key_s[:8] else: self._long_description = tubid_s self._short_description = tubid_s[:6] self.last_connect_time = None self.last_loss_time = None self.remote_host = None self.rref = None self._is_connected = False self._reconnector = None self._trigger_cb = None self._on_status_changed = ObserverList() def on_status_changed(self, status_changed): """ :param status_changed: a callable taking a single arg (the NativeStorageServer) that is notified when we become connected """ return self._on_status_changed.subscribe(status_changed) # Special methods used by copy.copy() and copy.deepcopy(). When those are # used in allmydata.immutable.filenode to copy CheckResults during # repair, we want it to treat the IServer instances as singletons, and # not attempt to duplicate them.. def __copy__(self): return self def __deepcopy__(self, memodict): return self def __repr__(self): return "<NativeStorageServer for %s>" % self.get_name() def get_serverid(self): return self._tubid # XXX replace with self.key_s def get_permutation_seed(self): return self._permutation_seed def get_version(self): if self.rref: return self.rref.version return None def get_name(self): # keep methodname short # TODO: decide who adds [] in the short description. It should # probably be the output side, not here. return self._short_description def get_longname(self): return self._long_description def get_lease_seed(self): return self._tubid def get_foolscap_write_enabler_seed(self): return self._tubid def get_nickname(self): return self.announcement["nickname"] def get_announcement(self): return self.announcement def get_remote_host(self): return self.remote_host def is_connected(self): return self._is_connected def get_last_connect_time(self): return self.last_connect_time def get_last_loss_time(self): return self.last_loss_time def get_last_received_data_time(self): if self.rref is None: return None else: return self.rref.getDataLastReceivedAt() def get_available_space(self): version = self.get_version() if version is None: return None protocol_v1_version = version.get( 'http://allmydata.org/tahoe/protocols/storage/v1', {}) available_space = protocol_v1_version.get('available-space') if available_space is None: available_space = protocol_v1_version.get( 'maximum-immutable-share-size', None) return available_space def start_connecting(self, trigger_cb): self._tub = Tub() for (name, value) in self._tub_options.items(): self._tub.setOption(name, value) # XXX todo: set tub handlers self._tub.setServiceParent(self) furl = str(self.announcement["anonymous-storage-FURL"]) self._trigger_cb = trigger_cb self._reconnector = self._tub.connectTo(furl, self._got_connection) def _got_connection(self, rref): lp = log.msg(format="got connection to %(name)s, getting versions", name=self.get_name(), facility="tahoe.storage_broker", umid="coUECQ") if self._trigger_cb: eventually(self._trigger_cb) default = self.VERSION_DEFAULTS d = add_version_to_remote_reference(rref, default) d.addCallback(self._got_versioned_service, lp) d.addCallback(lambda ign: self._on_status_changed.notify(self)) d.addErrback(log.err, format="storageclient._got_connection", name=self.get_name(), umid="Sdq3pg") def _got_versioned_service(self, rref, lp): log.msg(format="%(name)s provided version info %(version)s", name=self.get_name(), version=rref.version, facility="tahoe.storage_broker", umid="SWmJYg", level=log.NOISY, parent=lp) self.last_connect_time = time.time() self.remote_host = rref.getPeer() self.rref = rref self._is_connected = True rref.notifyOnDisconnect(self._lost) def get_rref(self): return self.rref def _lost(self): log.msg(format="lost connection to %(name)s", name=self.get_name(), facility="tahoe.storage_broker", umid="zbRllw") self.last_loss_time = time.time() # self.rref is now stale: all callRemote()s will get a # DeadReferenceError. We leave the stale reference in place so that # uploader/downloader code (which received this IServer through # get_connected_servers() or get_servers_for_psi()) can continue to # use s.get_rref().callRemote() and not worry about it being None. self._is_connected = False self.remote_host = None def stop_connecting(self): # used when this descriptor has been superceded by another self._reconnector.stopConnecting() def try_to_connect(self): # used when the broker wants us to hurry up self._reconnector.reset()