class LogTail:
def __init__(self, options):
self.options = options
def run(self, target_furl):
target_tubid = SturdyRef(target_furl).getTubRef().getTubID()
d = fireEventually(target_furl)
d.addCallback(self.start, target_tubid)
d.addErrback(self._error)
print("starting..")
reactor.run()
def _error(self, f):
print("ERROR", f)
reactor.stop()
def start(self, target_furl, target_tubid):
print("Connecting..")
self._tub = Tub()
self._tub.startService()
self._tub.connectTo(target_furl, self._got_logpublisher, target_tubid)
def _got_logpublisher(self, publisher, target_tubid):
d = publisher.callRemote("get_pid")
def _announce(pid_or_failure):
if isinstance(pid_or_failure, int):
print("Connected (to pid %d)" % pid_or_failure)
return pid_or_failure
else:
# the logport is probably foolscap-0.2.8 or earlier and
# doesn't offer get_pid()
print("Connected (unable to get pid)")
return None
d.addBoth(_announce)
publisher.notifyOnDisconnect(self._lost_logpublisher)
lp = LogPrinter(self.options, target_tubid)
def _ask_for_versions(pid):
d = publisher.callRemote("get_versions")
d.addCallback(lp.got_versions, pid)
return d
d.addCallback(_ask_for_versions)
catch_up = bool(self.options["catch-up"])
if catch_up:
d.addCallback(
lambda res: publisher.callRemote("subscribe_to_all", lp, True))
else:
# provide compatibility with foolscap-0.2.4 and earlier, which
# didn't accept a catchup= argument
d.addCallback(
lambda res: publisher.callRemote("subscribe_to_all", lp))
d.addErrback(self._error)
return d
def _lost_logpublisher(publisher):
print("Disconnected")
class LogTail:
def __init__(self, options):
self.options = options
def run(self, target_furl):
target_tubid = SturdyRef(target_furl).getTubRef().getTubID()
d = fireEventually(target_furl)
d.addCallback(self.start, target_tubid)
d.addErrback(self._error)
print "starting.."
reactor.run()
def _error(self, f):
print "ERROR", f
reactor.stop()
def start(self, target_furl, target_tubid):
print "Connecting.."
self._tub = Tub()
self._tub.startService()
self._tub.connectTo(target_furl, self._got_logpublisher, target_tubid)
def _got_logpublisher(self, publisher, target_tubid):
d = publisher.callRemote("get_pid")
def _announce(pid_or_failure):
if isinstance(pid_or_failure, int):
print "Connected (to pid %d)" % pid_or_failure
return pid_or_failure
else:
# the logport is probably foolscap-0.2.8 or earlier and
# doesn't offer get_pid()
print "Connected (unable to get pid)"
return None
d.addBoth(_announce)
publisher.notifyOnDisconnect(self._lost_logpublisher)
lp = LogPrinter(self.options, target_tubid)
def _ask_for_versions(pid):
d = publisher.callRemote("get_versions")
d.addCallback(lp.got_versions, pid)
return d
d.addCallback(_ask_for_versions)
catch_up = bool(self.options["catch-up"])
if catch_up:
d.addCallback(lambda res:
publisher.callRemote("subscribe_to_all", lp, True))
else:
# provide compatibility with foolscap-0.2.4 and earlier, which
# didn't accept a catchup= argument
d.addCallback(lambda res:
publisher.callRemote("subscribe_to_all", lp))
d.addErrback(self._error)
return d
def _lost_logpublisher(publisher):
print "Disconnected"
def test_queued_reconnector(self):
t1 = Tub()
bill_connections = []
barry_connections = []
t1.connectTo(self.bill_url, bill_connections.append)
t1.connectTo(self.barry_url, barry_connections.append)
def _check():
if len(bill_connections) >= 1 and len(barry_connections) >= 1:
return True
return False
d = self.poll(_check)
def _validate(res):
self.failUnless(isinstance(bill_connections[0], RemoteReference))
self.failUnless(isinstance(barry_connections[0], RemoteReference))
self.failIf(bill_connections[0] == barry_connections[0])
d.addCallback(_validate)
self.services.append(t1)
eventually(t1.startService)
return d
def test_queued_reconnector(self):
t1 = Tub()
bill_connections = []
barry_connections = []
t1.connectTo(self.bill_url, bill_connections.append)
t1.connectTo(self.barry_url, barry_connections.append)
def _check():
if len(bill_connections) >= 1 and len(barry_connections) >= 1:
return True
return False
d = self.poll(_check)
def _validate(res):
self.assertTrue(isinstance(bill_connections[0], RemoteReference))
self.assertTrue(isinstance(barry_connections[0], RemoteReference))
self.assertFalse(bill_connections[0] == barry_connections[0])
d.addCallback(_validate)
self.services.append(t1)
eventually(t1.startService)
return d
class Failed(unittest.TestCase):
def setUp(self):
self.services = []
def tearDown(self):
d = defer.DeferredList([s.stopService() for s in self.services])
d.addCallback(flushEventualQueue)
return d
@defer.inlineCallbacks
def test_bad_hints(self):
self.tubA = Tub()
self.tubA.startService()
self.services.append(self.tubA)
self.tubB = Tub()
self.tubB.startService()
self.services.append(self.tubB)
portnum = allocate_tcp_port()
self.tubB.listenOn("tcp:%d:interface=127.0.0.1" % portnum)
bad1 = "no-colon"
bad2 = "unknown:foo"
bad3 = "tcp:300.300.300.300:333"
self.tubB.setLocation(bad1, bad2, bad3)
target = HelperTarget("bob")
url = self.tubB.registerReference(target)
rc = self.tubA.connectTo(url, None)
ri = rc.getReconnectionInfo()
self.assertEqual(ri.state, "connecting")
d = defer.Deferred()
reactor.callLater(1.0, d.callback, None)
yield d
# now look at the details
ri = rc.getReconnectionInfo()
self.assertEqual(ri.state, "waiting")
ci = ri.connectionInfo
self.assertEqual(ci.connected, False)
self.assertEqual(ci.winningHint, None)
s = ci.connectorStatuses
self.assertEqual(set(s.keys()), set([bad1, bad2, bad3]))
self.assertEqual(s[bad1], "bad hint: no colon")
self.assertEqual(s[bad2], "bad hint: no handler registered")
self.assertIn("DNS lookup failed", s[bad3])
ch = ci.connectionHandlers
self.assertEqual(ch, {bad2: None, bad3: "tcp"})
class Failed(unittest.TestCase):
def setUp(self):
self.services = []
def tearDown(self):
d = defer.DeferredList([s.stopService() for s in self.services])
d.addCallback(flushEventualQueue)
return d
@defer.inlineCallbacks
def test_bad_hints(self):
self.tubA = Tub()
self.tubA.startService()
self.services.append(self.tubA)
self.tubB = Tub()
self.tubB.startService()
self.services.append(self.tubB)
portnum = allocate_tcp_port()
self.tubB.listenOn("tcp:%d:interface=127.0.0.1" % portnum)
bad1 = "no-colon"
bad2 = "unknown:foo"
bad3 = "tcp:300.300.300.300:333"
self.tubB.setLocation(bad1, bad2, bad3)
target = HelperTarget("bob")
url = self.tubB.registerReference(target)
rc = self.tubA.connectTo(url, None)
ri = rc.getReconnectionInfo()
self.assertEqual(ri.state, "connecting")
d = defer.Deferred()
reactor.callLater(1.0, d.callback, None)
yield d
# now look at the details
ri = rc.getReconnectionInfo()
self.assertEqual(ri.state, "waiting")
ci = ri.connectionInfo
self.assertEqual(ci.connected, False)
self.assertEqual(ci.winningHint, None)
s = ci.connectorStatuses
self.assertEqual(set(s.keys()), set([bad1, bad2, bad3]))
self.assertEqual(s[bad1], "bad hint: no colon")
self.assertEqual(s[bad2], "bad hint: no handler registered")
self.assertIn("DNS lookup failed", s[bad3])
ch = ci.connectionHandlers
self.assertEqual(ch, {bad2: None, bad3: "tcp"})
class NativeStorageServer(service.MultiService):
"""I hold information about a storage server that we want to connect to.
If we are connected, I hold the RemoteReference, their host address, and
the their version information. I remember information about when we were
last connected too, even if we aren't currently connected.
@ivar last_connect_time: when we last established a connection
@ivar last_loss_time: when we last lost a connection
@ivar version: the server's versiondict, from the most recent announcement
@ivar nickname: the server's self-reported nickname (unicode), same
@ivar rref: the RemoteReference, if connected, otherwise None
@ivar remote_host: the IAddress, if connected, otherwise None
"""
implements(IServer)
VERSION_DEFAULTS = {
"http://allmydata.org/tahoe/protocols/storage/v1" :
{ "maximum-immutable-share-size": 2**32 - 1,
"maximum-mutable-share-size": 2*1000*1000*1000, # maximum prior to v1.9.2
"tolerates-immutable-read-overrun": False,
"delete-mutable-shares-with-zero-length-writev": False,
"available-space": None,
},
"application-version": "unknown: no get_version()",
}
def __init__(self, key_s, ann, tub_options={}):
service.MultiService.__init__(self)
self.key_s = key_s
self.announcement = ann
self._tub_options = tub_options
assert "anonymous-storage-FURL" in ann, ann
furl = str(ann["anonymous-storage-FURL"])
m = re.match(r'pb://(\w+)@', furl)
assert m, furl
tubid_s = m.group(1).lower()
self._tubid = base32.a2b(tubid_s)
assert "permutation-seed-base32" in ann, ann
ps = base32.a2b(str(ann["permutation-seed-base32"]))
self._permutation_seed = ps
if key_s:
self._long_description = key_s
if key_s.startswith("v0-"):
# remove v0- prefix from abbreviated name
self._short_description = key_s[3:3+8]
else:
self._short_description = key_s[:8]
else:
self._long_description = tubid_s
self._short_description = tubid_s[:6]
self.last_connect_time = None
self.last_loss_time = None
self.remote_host = None
self.rref = None
self._is_connected = False
self._reconnector = None
self._trigger_cb = None
self._on_status_changed = ObserverList()
def on_status_changed(self, status_changed):
"""
:param status_changed: a callable taking a single arg (the
NativeStorageServer) that is notified when we become connected
"""
return self._on_status_changed.subscribe(status_changed)
# Special methods used by copy.copy() and copy.deepcopy(). When those are
# used in allmydata.immutable.filenode to copy CheckResults during
# repair, we want it to treat the IServer instances as singletons, and
# not attempt to duplicate them..
def __copy__(self):
return self
def __deepcopy__(self, memodict):
return self
def __repr__(self):
return "<NativeStorageServer for %s>" % self.get_name()
def get_serverid(self):
return self._tubid # XXX replace with self.key_s
def get_permutation_seed(self):
return self._permutation_seed
def get_version(self):
if self.rref:
return self.rref.version
return None
def get_name(self): # keep methodname short
# TODO: decide who adds [] in the short description. It should
# probably be the output side, not here.
return self._short_description
def get_longname(self):
return self._long_description
def get_lease_seed(self):
return self._tubid
def get_foolscap_write_enabler_seed(self):
return self._tubid
def get_nickname(self):
return self.announcement["nickname"]
def get_announcement(self):
return self.announcement
def get_remote_host(self):
return self.remote_host
def is_connected(self):
return self._is_connected
def get_last_connect_time(self):
return self.last_connect_time
def get_last_loss_time(self):
return self.last_loss_time
def get_last_received_data_time(self):
if self.rref is None:
return None
else:
return self.rref.getDataLastReceivedAt()
def get_available_space(self):
version = self.get_version()
if version is None:
return None
protocol_v1_version = version.get('http://allmydata.org/tahoe/protocols/storage/v1', {})
available_space = protocol_v1_version.get('available-space')
if available_space is None:
available_space = protocol_v1_version.get('maximum-immutable-share-size', None)
return available_space
def start_connecting(self, trigger_cb):
self._tub = Tub()
for (name, value) in self._tub_options.items():
self._tub.setOption(name, value)
self._tub.setServiceParent(self)
furl = str(self.announcement["anonymous-storage-FURL"])
self._trigger_cb = trigger_cb
self._reconnector = self._tub.connectTo(furl, self._got_connection)
def _got_connection(self, rref):
lp = log.msg(format="got connection to %(name)s, getting versions",
name=self.get_name(),
facility="tahoe.storage_broker", umid="coUECQ")
if self._trigger_cb:
eventually(self._trigger_cb)
default = self.VERSION_DEFAULTS
d = add_version_to_remote_reference(rref, default)
d.addCallback(self._got_versioned_service, lp)
d.addCallback(lambda ign: self._on_status_changed.notify(self))
d.addErrback(log.err, format="storageclient._got_connection",
name=self.get_name(), umid="Sdq3pg")
def _got_versioned_service(self, rref, lp):
log.msg(format="%(name)s provided version info %(version)s",
name=self.get_name(), version=rref.version,
facility="tahoe.storage_broker", umid="SWmJYg",
level=log.NOISY, parent=lp)
self.last_connect_time = time.time()
self.remote_host = rref.getPeer()
self.rref = rref
self._is_connected = True
rref.notifyOnDisconnect(self._lost)
def get_rref(self):
return self.rref
def _lost(self):
log.msg(format="lost connection to %(name)s", name=self.get_name(),
facility="tahoe.storage_broker", umid="zbRllw")
self.last_loss_time = time.time()
# self.rref is now stale: all callRemote()s will get a
# DeadReferenceError. We leave the stale reference in place so that
# uploader/downloader code (which received this IServer through
# get_connected_servers() or get_servers_for_psi()) can continue to
# use s.get_rref().callRemote() and not worry about it being None.
self._is_connected = False
self.remote_host = None
def stop_connecting(self):
# used when this descriptor has been superceded by another
self._reconnector.stopConnecting()
def try_to_connect(self):
# used when the broker wants us to hurry up
self._reconnector.reset()
class NativeStorageServer(service.MultiService):
"""I hold information about a storage server that we want to connect to.
If we are connected, I hold the RemoteReference, their host address, and
the their version information. I remember information about when we were
last connected too, even if we aren't currently connected.
@ivar last_connect_time: when we last established a connection
@ivar last_loss_time: when we last lost a connection
@ivar version: the server's versiondict, from the most recent announcement
@ivar nickname: the server's self-reported nickname (unicode), same
@ivar rref: the RemoteReference, if connected, otherwise None
@ivar remote_host: the IAddress, if connected, otherwise None
"""
implements(IServer)
VERSION_DEFAULTS = {
"http://allmydata.org/tahoe/protocols/storage/v1": {
"maximum-immutable-share-size": 2**32 - 1,
"maximum-mutable-share-size":
2 * 1000 * 1000 * 1000, # maximum prior to v1.9.2
"tolerates-immutable-read-overrun": False,
"delete-mutable-shares-with-zero-length-writev": False,
"available-space": None,
},
"application-version": "unknown: no get_version()",
}
def __init__(self, key_s, ann, tub_options={}, tub_handlers={}):
service.MultiService.__init__(self)
self.key_s = key_s
self.announcement = ann
self._tub_options = tub_options
self._tub_handlers = tub_handlers
assert "anonymous-storage-FURL" in ann, ann
furl = str(ann["anonymous-storage-FURL"])
m = re.match(r'pb://(\w+)@', furl)
assert m, furl
tubid_s = m.group(1).lower()
self._tubid = base32.a2b(tubid_s)
assert "permutation-seed-base32" in ann, ann
ps = base32.a2b(str(ann["permutation-seed-base32"]))
self._permutation_seed = ps
if key_s:
self._long_description = key_s
if key_s.startswith("v0-"):
# remove v0- prefix from abbreviated name
self._short_description = key_s[3:3 + 8]
else:
self._short_description = key_s[:8]
else:
self._long_description = tubid_s
self._short_description = tubid_s[:6]
self.last_connect_time = None
self.last_loss_time = None
self.remote_host = None
self.rref = None
self._is_connected = False
self._reconnector = None
self._trigger_cb = None
self._on_status_changed = ObserverList()
def on_status_changed(self, status_changed):
"""
:param status_changed: a callable taking a single arg (the
NativeStorageServer) that is notified when we become connected
"""
return self._on_status_changed.subscribe(status_changed)
# Special methods used by copy.copy() and copy.deepcopy(). When those are
# used in allmydata.immutable.filenode to copy CheckResults during
# repair, we want it to treat the IServer instances as singletons, and
# not attempt to duplicate them..
def __copy__(self):
return self
def __deepcopy__(self, memodict):
return self
def __repr__(self):
return "<NativeStorageServer for %s>" % self.get_name()
def get_serverid(self):
return self._tubid # XXX replace with self.key_s
def get_permutation_seed(self):
return self._permutation_seed
def get_version(self):
if self.rref:
return self.rref.version
return None
def get_name(self): # keep methodname short
# TODO: decide who adds [] in the short description. It should
# probably be the output side, not here.
return self._short_description
def get_longname(self):
return self._long_description
def get_lease_seed(self):
return self._tubid
def get_foolscap_write_enabler_seed(self):
return self._tubid
def get_nickname(self):
return self.announcement["nickname"]
def get_announcement(self):
return self.announcement
def get_remote_host(self):
return self.remote_host
def is_connected(self):
return self._is_connected
def get_last_connect_time(self):
return self.last_connect_time
def get_last_loss_time(self):
return self.last_loss_time
def get_last_received_data_time(self):
if self.rref is None:
return None
else:
return self.rref.getDataLastReceivedAt()
def get_available_space(self):
version = self.get_version()
if version is None:
return None
protocol_v1_version = version.get(
'http://allmydata.org/tahoe/protocols/storage/v1', {})
available_space = protocol_v1_version.get('available-space')
if available_space is None:
available_space = protocol_v1_version.get(
'maximum-immutable-share-size', None)
return available_space
def start_connecting(self, trigger_cb):
self._tub = Tub()
for (name, value) in self._tub_options.items():
self._tub.setOption(name, value)
# XXX todo: set tub handlers
self._tub.setServiceParent(self)
furl = str(self.announcement["anonymous-storage-FURL"])
self._trigger_cb = trigger_cb
self._reconnector = self._tub.connectTo(furl, self._got_connection)
def _got_connection(self, rref):
lp = log.msg(format="got connection to %(name)s, getting versions",
name=self.get_name(),
facility="tahoe.storage_broker",
umid="coUECQ")
if self._trigger_cb:
eventually(self._trigger_cb)
default = self.VERSION_DEFAULTS
d = add_version_to_remote_reference(rref, default)
d.addCallback(self._got_versioned_service, lp)
d.addCallback(lambda ign: self._on_status_changed.notify(self))
d.addErrback(log.err,
format="storageclient._got_connection",
name=self.get_name(),
umid="Sdq3pg")
def _got_versioned_service(self, rref, lp):
log.msg(format="%(name)s provided version info %(version)s",
name=self.get_name(),
version=rref.version,
facility="tahoe.storage_broker",
umid="SWmJYg",
level=log.NOISY,
parent=lp)
self.last_connect_time = time.time()
self.remote_host = rref.getPeer()
self.rref = rref
self._is_connected = True
rref.notifyOnDisconnect(self._lost)
def get_rref(self):
return self.rref
def _lost(self):
log.msg(format="lost connection to %(name)s",
name=self.get_name(),
facility="tahoe.storage_broker",
umid="zbRllw")
self.last_loss_time = time.time()
# self.rref is now stale: all callRemote()s will get a
# DeadReferenceError. We leave the stale reference in place so that
# uploader/downloader code (which received this IServer through
# get_connected_servers() or get_servers_for_psi()) can continue to
# use s.get_rref().callRemote() and not worry about it being None.
self._is_connected = False
self.remote_host = None
def stop_connecting(self):
# used when this descriptor has been superceded by another
self._reconnector.stopConnecting()
def try_to_connect(self):
# used when the broker wants us to hurry up
self._reconnector.reset()
