def _setup_download(self):
self._started = time.time()
self._status.set_status("Retrieving Shares")
# how many shares do we need?
(seqnum, root_hash, IV, segsize, datalength, k, N, prefix, offsets_tuple) = self.verinfo
# first, which servers can we use?
versionmap = self.servermap.make_versionmap()
shares = versionmap[self.verinfo]
# this sharemap is consumed as we decide to send requests
self.remaining_sharemap = DictOfSets()
for (shnum, server, timestamp) in shares:
self.remaining_sharemap.add(shnum, server)
# If the servermap update fetched anything, it fetched at least 1
# KiB, so we ask for that much.
# TODO: Change the cache methods to allow us to fetch all of the
# data that they have, then change this method to do that.
any_cache = self._node._read_from_cache(self.verinfo, shnum, 0, 1000)
reader = MDMFSlotReadProxy(server.get_rref(), self._storage_index, shnum, any_cache)
reader.server = server
self.readers[shnum] = reader
assert len(self.remaining_sharemap) >= k
self.shares = {} # maps shnum to validated blocks
self._active_readers = [] # list of active readers for this dl.
self._block_hash_trees = {} # shnum => hashtree
# We need one share hash tree for the entire file; its leaves
# are the roots of the block hash trees for the shares that
# comprise it, and its root is in the verinfo.
self.share_hash_tree = hashtree.IncompleteHashTree(N)
self.share_hash_tree.set_hashes({0: root_hash})
def _setup_download(self):
self._started = time.time()
self._status.set_status("Retrieving Shares")
# how many shares do we need?
(seqnum,
root_hash,
IV,
segsize,
datalength,
k,
N,
prefix,
offsets_tuple) = self.verinfo
# first, which servers can we use?
versionmap = self.servermap.make_versionmap()
shares = versionmap[self.verinfo]
# this sharemap is consumed as we decide to send requests
self.remaining_sharemap = DictOfSets()
for (shnum, server, timestamp) in shares:
self.remaining_sharemap.add(shnum, server)
# Reuse the SlotReader from the servermap.
key = (self.verinfo, server.get_serverid(),
self._storage_index, shnum)
if key in self.servermap.proxies:
reader = self.servermap.proxies[key]
else:
reader = MDMFSlotReadProxy(server.get_rref(),
self._storage_index, shnum, None)
reader.server = server
self.readers[shnum] = reader
if len(self.remaining_sharemap) < k:
self._raise_notenoughshareserror()
self.shares = {} # maps shnum to validated blocks
self._active_readers = [] # list of active readers for this dl.
self._block_hash_trees = {} # shnum => hashtree
for i in xrange(self._total_shares):
# So we don't have to do this later.
self._block_hash_trees[i] = hashtree.IncompleteHashTree(self._num_segments)
# We need one share hash tree for the entire file; its leaves
# are the roots of the block hash trees for the shares that
# comprise it, and its root is in the verinfo.
self.share_hash_tree = hashtree.IncompleteHashTree(N)
self.share_hash_tree.set_hashes({0: root_hash})
def corrupt(res, s, offset, shnums_to_corrupt=None, offset_offset=0):
# if shnums_to_corrupt is None, corrupt all shares. Otherwise it is a
# list of shnums to corrupt.
ds = []
for peerid in s._peers:
shares = s._peers[peerid]
for shnum in shares:
if (shnums_to_corrupt is not None
and shnum not in shnums_to_corrupt):
continue
data = shares[shnum]
# We're feeding the reader all of the share data, so it
# won't need to use the rref that we didn't provide, nor the
# storage index that we didn't provide. We do this because
# the reader will work for both MDMF and SDMF.
reader = MDMFSlotReadProxy(None, None, shnum, data)
# We need to get the offsets for the next part.
d = reader.get_verinfo()
def _do_corruption(verinfo, data, shnum, shares):
(seqnum,
root_hash,
IV,
segsize,
datalen,
k, n, prefix, o) = verinfo
if isinstance(offset, tuple):
offset1, offset2 = offset
else:
offset1 = offset
offset2 = 0
if offset1 == "pubkey" and IV:
real_offset = 107
elif offset1 in o:
real_offset = o[offset1]
else:
real_offset = offset1
real_offset = int(real_offset) + offset2 + offset_offset
assert isinstance(real_offset, int), offset
if offset1 == 0: # verbyte
f = add_two
else:
f = flip_bit
shares[shnum] = f(data, real_offset)
d.addCallback(_do_corruption, data, shnum, shares)
ds.append(d)
dl = defer.DeferredList(ds)
dl.addCallback(lambda ignored: res)
return dl
def corrupt(res, s, offset, shnums_to_corrupt=None, offset_offset=0):
# if shnums_to_corrupt is None, corrupt all shares. Otherwise it is a
# list of shnums to corrupt.
ds = []
for peerid in s._peers:
shares = s._peers[peerid]
for shnum in shares:
if (shnums_to_corrupt is not None
and shnum not in shnums_to_corrupt):
continue
data = shares[shnum]
# We're feeding the reader all of the share data, so it
# won't need to use the rref that we didn't provide, nor the
# storage index that we didn't provide. We do this because
# the reader will work for both MDMF and SDMF.
reader = MDMFSlotReadProxy(None, None, shnum, data)
# We need to get the offsets for the next part.
d = reader.get_verinfo()
def _do_corruption(verinfo, data, shnum, shares):
(seqnum,
root_hash,
IV,
segsize,
datalen,
k, n, prefix, o) = verinfo
if isinstance(offset, tuple):
offset1, offset2 = offset
else:
offset1 = offset
offset2 = 0
if offset1 == "pubkey" and IV:
real_offset = 107
elif offset1 in o:
real_offset = o[offset1]
else:
real_offset = offset1
real_offset = int(real_offset) + offset2 + offset_offset
assert isinstance(real_offset, int), offset
if offset1 == 0: # verbyte
f = add_two
else:
f = flip_bit
shares[shnum] = f(data, real_offset)
d.addCallback(_do_corruption, data, shnum, shares)
ds.append(d)
dl = defer.DeferredList(ds)
dl.addCallback(lambda ignored: res)
return dl
def _setup_download(self):
self._started = time.time()
self._status.set_status("Retrieving Shares")
# how many shares do we need?
(seqnum, root_hash, IV, segsize, datalength, k, N, prefix,
offsets_tuple) = self.verinfo
# first, which servers can we use?
versionmap = self.servermap.make_versionmap()
shares = versionmap[self.verinfo]
# this sharemap is consumed as we decide to send requests
self.remaining_sharemap = DictOfSets()
for (shnum, server, timestamp) in shares:
self.remaining_sharemap.add(shnum, server)
# If the servermap update fetched anything, it fetched at least 1
# KiB, so we ask for that much.
# TODO: Change the cache methods to allow us to fetch all of the
# data that they have, then change this method to do that.
any_cache = self._node._read_from_cache(self.verinfo, shnum, 0,
1000)
reader = MDMFSlotReadProxy(server.get_rref(), self._storage_index,
shnum, any_cache)
reader.server = server
self.readers[shnum] = reader
assert len(self.remaining_sharemap) >= k
self.shares = {} # maps shnum to validated blocks
self._active_readers = [] # list of active readers for this dl.
self._block_hash_trees = {} # shnum => hashtree
# We need one share hash tree for the entire file; its leaves
# are the roots of the block hash trees for the shares that
# comprise it, and its root is in the verinfo.
self.share_hash_tree = hashtree.IncompleteHashTree(N)
self.share_hash_tree.set_hashes({0: root_hash})
