def _maybe_create_download_node(self):
if not self._download_status:
ds = DownloadStatus(self._verifycap.storage_index,
self._verifycap.size)
if self._history:
self._history.add_download(ds)
self._download_status = ds
if self._node is None:
self._node = DownloadNode(self._verifycap, self._storage_broker,
self._secret_holder, self._terminator,
self._history, self._download_status)
def _maybe_create_download_node(self):
if not self._download_status:
ds = DownloadStatus(self._verifycap.storage_index,
self._verifycap.size)
if self._history:
self._history.add_download(ds)
self._download_status = ds
if self._node is None:
self._node = DownloadNode(self._verifycap, self._storage_broker,
self._secret_holder,
self._terminator,
self._history, self._download_status)
class CiphertextFileNode:
def __init__(self, verifycap, storage_broker, secret_holder,
terminator, history):
assert isinstance(verifycap, uri.CHKFileVerifierURI)
self._verifycap = verifycap
self._storage_broker = storage_broker
self._secret_holder = secret_holder
self._terminator = terminator
self._history = history
self._download_status = None
self._node = None # created lazily, on read()
def _maybe_create_download_node(self):
if not self._download_status:
ds = DownloadStatus(self._verifycap.storage_index,
self._verifycap.size)
if self._history:
self._history.add_download(ds)
self._download_status = ds
if self._node is None:
self._node = DownloadNode(self._verifycap, self._storage_broker,
self._secret_holder,
self._terminator,
self._history, self._download_status)
def read(self, consumer, offset=0, size=None):
"""I am the main entry point, from which FileNode.read() can get
data. I feed the consumer with the desired range of ciphertext. I
return a Deferred that fires (with the consumer) when the read is
finished."""
self._maybe_create_download_node()
actual_size = size
if actual_size is None:
actual_size = self._verifycap.size - offset
read_ev = self._download_status.add_read_event(offset, actual_size,
now())
if IDownloadStatusHandlingConsumer.providedBy(consumer):
consumer.set_download_status_read_event(read_ev)
return self._node.read(consumer, offset, size, read_ev)
def get_segment(self, segnum):
"""Begin downloading a segment. I return a tuple (d, c): 'd' is a
Deferred that fires with (offset,data) when the desired segment is
available, and c is an object on which c.cancel() can be called to
disavow interest in the segment (after which 'd' will never fire).
You probably need to know the segment size before calling this,
unless you want the first few bytes of the file. If you ask for a
segment number which turns out to be too large, the Deferred will
errback with BadSegmentNumberError.
The Deferred fires with the offset of the first byte of the data
segment, so that you can call get_segment() before knowing the
segment size, and still know which data you received.
"""
self._maybe_create_download_node()
return self._node.get_segment(segnum)
def get_segment_size(self):
# return a Deferred that fires with the file's real segment size
self._maybe_create_download_node()
return self._node.get_segsize()
def get_storage_index(self):
return self._verifycap.storage_index
def get_verify_cap(self):
return self._verifycap
def get_size(self):
return self._verifycap.size
def raise_error(self):
pass
def check_and_repair(self, monitor, verify=False, add_lease=False):
verifycap = self._verifycap
storage_index = verifycap.storage_index
sb = self._storage_broker
servers = sb.get_all_servers()
sh = self._secret_holder
c = Checker(verifycap=verifycap, servers=servers,
verify=verify, add_lease=add_lease, secret_holder=sh,
monitor=monitor)
d = c.start()
def _maybe_repair(cr):
crr = CheckAndRepairResults(storage_index)
crr.pre_repair_results = cr
if cr.is_healthy():
crr.post_repair_results = cr
return defer.succeed(crr)
else:
crr.repair_attempted = True
crr.repair_successful = False # until proven successful
def _gather_repair_results(ur):
assert IUploadResults.providedBy(ur), ur
# clone the cr (check results) to form the basis of the
# prr (post-repair results)
prr = CheckResults(cr.uri, cr.storage_index)
prr.data = copy.deepcopy(cr.data)
sm = prr.data['sharemap']
assert isinstance(sm, DictOfSets), sm
sm.update(ur.sharemap)
servers_responding = set(prr.data['servers-responding'])
servers_responding.union(ur.sharemap.iterkeys())
prr.data['servers-responding'] = list(servers_responding)
prr.data['count-shares-good'] = len(sm)
prr.data['count-good-share-hosts'] = len(sm)
is_healthy = bool(len(sm) >= verifycap.total_shares)
is_recoverable = bool(len(sm) >= verifycap.needed_shares)
prr.set_healthy(is_healthy)
prr.set_recoverable(is_recoverable)
crr.repair_successful = is_healthy
prr.set_needs_rebalancing(len(sm) >= verifycap.total_shares)
crr.post_repair_results = prr
return crr
def _repair_error(f):
# as with mutable repair, I'm not sure if I want to pass
# through a failure or not. TODO
crr.repair_successful = False
crr.repair_failure = f
return f
r = Repairer(self, storage_broker=sb, secret_holder=sh,
monitor=monitor)
d = r.start()
d.addCallbacks(_gather_repair_results, _repair_error)
return d
d.addCallback(_maybe_repair)
return d
def check(self, monitor, verify=False, add_lease=False):
verifycap = self._verifycap
sb = self._storage_broker
servers = sb.get_all_servers()
sh = self._secret_holder
v = Checker(verifycap=verifycap, servers=servers,
verify=verify, add_lease=add_lease, secret_holder=sh,
monitor=monitor)
return v.start()
class CiphertextFileNode:
def __init__(self, verifycap, storage_broker, secret_holder, terminator,
history):
assert isinstance(verifycap, uri.CHKFileVerifierURI)
self._verifycap = verifycap
self._storage_broker = storage_broker
self._secret_holder = secret_holder
self._terminator = terminator
self._history = history
self._download_status = None
self._node = None # created lazily, on read()
def _maybe_create_download_node(self):
if not self._download_status:
ds = DownloadStatus(self._verifycap.storage_index,
self._verifycap.size)
if self._history:
self._history.add_download(ds)
self._download_status = ds
if self._node is None:
self._node = DownloadNode(self._verifycap, self._storage_broker,
self._secret_holder, self._terminator,
self._history, self._download_status)
def read(self, consumer, offset=0, size=None):
"""I am the main entry point, from which FileNode.read() can get
data. I feed the consumer with the desired range of ciphertext. I
return a Deferred that fires (with the consumer) when the read is
finished."""
self._maybe_create_download_node()
return self._node.read(consumer, offset, size)
def get_segment(self, segnum):
"""Begin downloading a segment. I return a tuple (d, c): 'd' is a
Deferred that fires with (offset,data) when the desired segment is
available, and c is an object on which c.cancel() can be called to
disavow interest in the segment (after which 'd' will never fire).
You probably need to know the segment size before calling this,
unless you want the first few bytes of the file. If you ask for a
segment number which turns out to be too large, the Deferred will
errback with BadSegmentNumberError.
The Deferred fires with the offset of the first byte of the data
segment, so that you can call get_segment() before knowing the
segment size, and still know which data you received.
"""
self._maybe_create_download_node()
return self._node.get_segment(segnum)
def get_segment_size(self):
# return a Deferred that fires with the file's real segment size
self._maybe_create_download_node()
return self._node.get_segsize()
def get_storage_index(self):
return self._verifycap.storage_index
def get_verify_cap(self):
return self._verifycap
def get_size(self):
return self._verifycap.size
def raise_error(self):
pass
def is_mutable(self):
return False
def check_and_repair(self, monitor, verify=False, add_lease=False):
c = Checker(verifycap=self._verifycap,
servers=self._storage_broker.get_connected_servers(),
verify=verify,
add_lease=add_lease,
secret_holder=self._secret_holder,
monitor=monitor)
d = c.start()
d.addCallback(self._maybe_repair, monitor)
return d
def _maybe_repair(self, cr, monitor):
crr = CheckAndRepairResults(self._verifycap.storage_index)
crr.pre_repair_results = cr
if cr.is_healthy():
crr.post_repair_results = cr
return defer.succeed(crr)
crr.repair_attempted = True
crr.repair_successful = False # until proven successful
def _repair_error(f):
# as with mutable repair, I'm not sure if I want to pass
# through a failure or not. TODO
crr.repair_successful = False
crr.repair_failure = f
return f
r = Repairer(self,
storage_broker=self._storage_broker,
secret_holder=self._secret_holder,
monitor=monitor)
d = r.start()
d.addCallbacks(self._gather_repair_results,
_repair_error,
callbackArgs=(
cr,
crr,
))
return d
def _gather_repair_results(self, ur, cr, crr):
assert IUploadResults.providedBy(ur), ur
# clone the cr (check results) to form the basis of the
# prr (post-repair results)
verifycap = self._verifycap
servers_responding = set(cr.get_servers_responding())
sm = DictOfSets()
assert isinstance(cr.get_sharemap(), DictOfSets)
for shnum, servers in cr.get_sharemap().items():
for server in servers:
sm.add(shnum, server)
for shnum, servers in ur.get_sharemap().items():
for server in servers:
sm.add(shnum, server)
servers_responding.add(server)
servers_responding = sorted(servers_responding)
good_hosts = len(reduce(set.union, sm.values(), set()))
is_healthy = bool(len(sm) >= verifycap.total_shares)
is_recoverable = bool(len(sm) >= verifycap.needed_shares)
count_happiness = servers_of_happiness(sm)
prr = CheckResults(
cr.get_uri(),
cr.get_storage_index(),
healthy=is_healthy,
recoverable=is_recoverable,
count_happiness=count_happiness,
count_shares_needed=verifycap.needed_shares,
count_shares_expected=verifycap.total_shares,
count_shares_good=len(sm),
count_good_share_hosts=good_hosts,
count_recoverable_versions=int(is_recoverable),
count_unrecoverable_versions=int(not is_recoverable),
servers_responding=list(servers_responding),
sharemap=sm,
count_wrong_shares=0, # no such thing as wrong, for immutable
list_corrupt_shares=cr.get_corrupt_shares(),
count_corrupt_shares=len(cr.get_corrupt_shares()),
list_incompatible_shares=cr.get_incompatible_shares(),
count_incompatible_shares=len(cr.get_incompatible_shares()),
summary="",
report=[],
share_problems=[],
servermap=None)
crr.repair_successful = is_healthy
crr.post_repair_results = prr
return crr
def check(self, monitor, verify=False, add_lease=False):
verifycap = self._verifycap
sb = self._storage_broker
servers = sb.get_connected_servers()
sh = self._secret_holder
v = Checker(verifycap=verifycap,
servers=servers,
verify=verify,
add_lease=add_lease,
secret_holder=sh,
monitor=monitor)
return v.start()
class CiphertextFileNode:
def __init__(self, verifycap, storage_broker, secret_holder, terminator,
history):
assert isinstance(verifycap, uri.CHKFileVerifierURI)
self._verifycap = verifycap
self._storage_broker = storage_broker
self._secret_holder = secret_holder
self._terminator = terminator
self._history = history
self._download_status = None
self._node = None # created lazily, on read()
def _maybe_create_download_node(self):
if not self._download_status:
ds = DownloadStatus(self._verifycap.storage_index,
self._verifycap.size)
if self._history:
self._history.add_download(ds)
self._download_status = ds
if self._node is None:
self._node = DownloadNode(self._verifycap, self._storage_broker,
self._secret_holder, self._terminator,
self._history, self._download_status)
def read(self, consumer, offset=0, size=None):
"""I am the main entry point, from which FileNode.read() can get
data. I feed the consumer with the desired range of ciphertext. I
return a Deferred that fires (with the consumer) when the read is
finished."""
self._maybe_create_download_node()
actual_size = size
if actual_size is None:
actual_size = self._verifycap.size - offset
read_ev = self._download_status.add_read_event(offset, actual_size,
now())
if IDownloadStatusHandlingConsumer.providedBy(consumer):
consumer.set_download_status_read_event(read_ev)
return self._node.read(consumer, offset, size, read_ev)
def get_segment(self, segnum):
"""Begin downloading a segment. I return a tuple (d, c): 'd' is a
Deferred that fires with (offset,data) when the desired segment is
available, and c is an object on which c.cancel() can be called to
disavow interest in the segment (after which 'd' will never fire).
You probably need to know the segment size before calling this,
unless you want the first few bytes of the file. If you ask for a
segment number which turns out to be too large, the Deferred will
errback with BadSegmentNumberError.
The Deferred fires with the offset of the first byte of the data
segment, so that you can call get_segment() before knowing the
segment size, and still know which data you received.
"""
self._maybe_create_download_node()
return self._node.get_segment(segnum)
def get_segment_size(self):
# return a Deferred that fires with the file's real segment size
self._maybe_create_download_node()
return self._node.get_segsize()
def get_storage_index(self):
return self._verifycap.storage_index
def get_verify_cap(self):
return self._verifycap
def get_size(self):
return self._verifycap.size
def raise_error(self):
pass
def check_and_repair(self, monitor, verify=False, add_lease=False):
verifycap = self._verifycap
storage_index = verifycap.storage_index
sb = self._storage_broker
servers = sb.get_connected_servers()
sh = self._secret_holder
c = Checker(verifycap=verifycap,
servers=servers,
verify=verify,
add_lease=add_lease,
secret_holder=sh,
monitor=monitor)
d = c.start()
def _maybe_repair(cr):
crr = CheckAndRepairResults(storage_index)
crr.pre_repair_results = cr
if cr.is_healthy():
crr.post_repair_results = cr
return defer.succeed(crr)
else:
crr.repair_attempted = True
crr.repair_successful = False # until proven successful
def _gather_repair_results(ur):
assert IUploadResults.providedBy(ur), ur
# clone the cr (check results) to form the basis of the
# prr (post-repair results)
prr = CheckResults(cr.uri, cr.storage_index)
prr.data = copy.deepcopy(cr.data)
sm = prr.data['sharemap']
assert isinstance(sm, DictOfSets), sm
sm.update(ur.sharemap)
servers_responding = set(prr.data['servers-responding'])
servers_responding.union(ur.sharemap.iterkeys())
prr.data['servers-responding'] = list(servers_responding)
prr.data['count-shares-good'] = len(sm)
prr.data['count-good-share-hosts'] = len(sm)
is_healthy = bool(len(sm) >= verifycap.total_shares)
is_recoverable = bool(len(sm) >= verifycap.needed_shares)
prr.set_healthy(is_healthy)
prr.set_recoverable(is_recoverable)
crr.repair_successful = is_healthy
prr.set_needs_rebalancing(
len(sm) >= verifycap.total_shares)
crr.post_repair_results = prr
return crr
def _repair_error(f):
# as with mutable repair, I'm not sure if I want to pass
# through a failure or not. TODO
crr.repair_successful = False
crr.repair_failure = f
return f
r = Repairer(self,
storage_broker=sb,
secret_holder=sh,
monitor=monitor)
d = r.start()
d.addCallbacks(_gather_repair_results, _repair_error)
return d
d.addCallback(_maybe_repair)
return d
def check(self, monitor, verify=False, add_lease=False):
verifycap = self._verifycap
sb = self._storage_broker
servers = sb.get_connected_servers()
sh = self._secret_holder
v = Checker(verifycap=verifycap,
servers=servers,
verify=verify,
add_lease=add_lease,
secret_holder=sh,
monitor=monitor)
return v.start()
class CiphertextFileNode:
def __init__(self, verifycap, storage_broker, secret_holder,
terminator, history):
assert isinstance(verifycap, uri.CHKFileVerifierURI)
self._verifycap = verifycap
self._storage_broker = storage_broker
self._secret_holder = secret_holder
self._terminator = terminator
self._history = history
self._download_status = None
self._node = None # created lazily, on read()
def _maybe_create_download_node(self):
if not self._download_status:
ds = DownloadStatus(self._verifycap.storage_index,
self._verifycap.size)
if self._history:
self._history.add_download(ds)
self._download_status = ds
if self._node is None:
self._node = DownloadNode(self._verifycap, self._storage_broker,
self._secret_holder,
self._terminator,
self._history, self._download_status)
def read(self, consumer, offset=0, size=None):
"""I am the main entry point, from which FileNode.read() can get
data. I feed the consumer with the desired range of ciphertext. I
return a Deferred that fires (with the consumer) when the read is
finished."""
self._maybe_create_download_node()
return self._node.read(consumer, offset, size)
def get_segment(self, segnum):
"""Begin downloading a segment. I return a tuple (d, c): 'd' is a
Deferred that fires with (offset,data) when the desired segment is
available, and c is an object on which c.cancel() can be called to
disavow interest in the segment (after which 'd' will never fire).
You probably need to know the segment size before calling this,
unless you want the first few bytes of the file. If you ask for a
segment number which turns out to be too large, the Deferred will
errback with BadSegmentNumberError.
The Deferred fires with the offset of the first byte of the data
segment, so that you can call get_segment() before knowing the
segment size, and still know which data you received.
"""
self._maybe_create_download_node()
return self._node.get_segment(segnum)
def get_segment_size(self):
# return a Deferred that fires with the file's real segment size
self._maybe_create_download_node()
return self._node.get_segsize()
def get_storage_index(self):
return self._verifycap.storage_index
def get_verify_cap(self):
return self._verifycap
def get_size(self):
return self._verifycap.size
def raise_error(self):
pass
def is_mutable(self):
return False
def check_and_repair(self, monitor, verify=False, add_lease=False):
c = Checker(verifycap=self._verifycap,
servers=self._storage_broker.get_connected_servers(),
verify=verify, add_lease=add_lease,
secret_holder=self._secret_holder,
monitor=monitor)
d = c.start()
d.addCallback(self._maybe_repair, monitor)
return d
def _maybe_repair(self, cr, monitor):
crr = CheckAndRepairResults(self._verifycap.storage_index)
crr.pre_repair_results = cr
if cr.is_healthy():
crr.post_repair_results = cr
return defer.succeed(crr)
crr.repair_attempted = True
crr.repair_successful = False # until proven successful
def _repair_error(f):
# as with mutable repair, I'm not sure if I want to pass
# through a failure or not. TODO
crr.repair_successful = False
crr.repair_failure = f
return f
r = Repairer(self, storage_broker=self._storage_broker,
secret_holder=self._secret_holder,
monitor=monitor)
d = r.start()
d.addCallbacks(self._gather_repair_results, _repair_error,
callbackArgs=(cr, crr,))
return d
def _gather_repair_results(self, ur, cr, crr):
assert IUploadResults.providedBy(ur), ur
# clone the cr (check results) to form the basis of the
# prr (post-repair results)
verifycap = self._verifycap
servers_responding = set(cr.get_servers_responding())
sm = DictOfSets()
assert isinstance(cr.get_sharemap(), DictOfSets)
for shnum, servers in cr.get_sharemap().items():
for server in servers:
sm.add(shnum, server)
for shnum, servers in ur.get_sharemap().items():
for server in servers:
sm.add(shnum, server)
servers_responding.add(server)
servers_responding = sorted(servers_responding)
good_hosts = len(reduce(set.union, sm.values(), set()))
is_healthy = bool(len(sm) >= verifycap.total_shares)
is_recoverable = bool(len(sm) >= verifycap.needed_shares)
needs_rebalancing = bool(len(sm) >= verifycap.total_shares)
prr = CheckResults(cr.get_uri(), cr.get_storage_index(),
healthy=is_healthy, recoverable=is_recoverable,
needs_rebalancing=needs_rebalancing,
count_shares_needed=verifycap.needed_shares,
count_shares_expected=verifycap.total_shares,
count_shares_good=len(sm),
count_good_share_hosts=good_hosts,
count_recoverable_versions=int(is_recoverable),
count_unrecoverable_versions=int(not is_recoverable),
servers_responding=list(servers_responding),
sharemap=sm,
count_wrong_shares=0, # no such thing as wrong, for immutable
list_corrupt_shares=cr.get_corrupt_shares(),
count_corrupt_shares=len(cr.get_corrupt_shares()),
list_incompatible_shares=cr.get_incompatible_shares(),
count_incompatible_shares=len(cr.get_incompatible_shares()),
summary="",
report=[],
share_problems=[],
servermap=None)
crr.repair_successful = is_healthy
crr.post_repair_results = prr
return crr
def check(self, monitor, verify=False, add_lease=False):
verifycap = self._verifycap
sb = self._storage_broker
servers = sb.get_connected_servers()
sh = self._secret_holder
v = Checker(verifycap=verifycap, servers=servers,
verify=verify, add_lease=add_lease, secret_holder=sh,
monitor=monitor)
return v.start()
