def get_size(self):
if isinstance(self.my_uri, uri.LiteralFileURI):
return self.my_uri.get_size()
try:
data = self.all_contents[self.my_uri.to_string()]
except KeyError, le:
raise NotEnoughSharesError(le, 0, 3)
def _read(self, ignored, consumer, offset, size):
if isinstance(self.my_uri, uri.LiteralFileURI):
data = self.my_uri.data
else:
if self.my_uri.to_string() not in self.all_contents:
raise NotEnoughSharesError(None, 0, 3)
data = self.all_contents[self.my_uri.to_string()]
start = offset
if size is not None:
end = offset + size
else:
end = len(data)
consumer.write(data[start:end])
return consumer
def _raise_notenoughshareserror(self):
"""
I am called when there are not enough active servers left to complete
the download. After making some useful logging statements, I throw an
exception to that effect to the caller of this Retrieve object through
self._done_deferred.
"""
format = ("ran out of servers: "
"have %(have)d of %(total)d segments; "
"found %(bad)d bad shares; "
"have %(remaining)d remaining shares of the right version; "
"encoding %(k)d-of-%(n)d")
args = {"have": self._current_segment,
"total": self._num_segments,
"need": self._last_segment,
"k": self._required_shares,
"n": self._total_shares,
"bad": len(self._bad_shares),
"remaining": len(self.remaining_sharemap),
}
raise NotEnoughSharesError("%s, last failure: %s" %
(format % args, str(self._last_failure)))
def _download_best_version(self, ignored=None, progress=None):
if isinstance(self.my_uri, uri.LiteralFileURI):
return self.my_uri.data
if self.storage_index not in self.all_contents:
raise NotEnoughSharesError(None, 0, 3)
return self.all_contents[self.storage_index]
def _maybe_send_more_queries(self, k):
# we don't have enough shares yet. Should we send out more queries?
# There are some number of queries outstanding, each for a single
# share. If we can generate 'needed_shares' additional queries, we do
# so. If we can't, then we know this file is a goner, and we raise
# NotEnoughSharesError.
self.log(format=("_maybe_send_more_queries, have=%(have)d, k=%(k)d, "
"outstanding=%(outstanding)d"),
have=len(self.shares),
k=k,
outstanding=len(self._outstanding_queries),
level=log.NOISY)
remaining_shares = k - len(self.shares)
needed = remaining_shares - len(self._outstanding_queries)
if not needed:
# we have enough queries in flight already
# TODO: but if they've been in flight for a long time, and we
# have reason to believe that new queries might respond faster
# (i.e. we've seen other queries come back faster, then consider
# sending out new queries. This could help with peers which have
# silently gone away since the servermap was updated, for which
# we're still waiting for the 15-minute TCP disconnect to happen.
self.log("enough queries are in flight, no more are needed",
level=log.NOISY)
return
outstanding_shnums = set([
shnum
for (peerid, shnum, started) in self._outstanding_queries.values()
])
# prefer low-numbered shares, they are more likely to be primary
available_shnums = sorted(self.remaining_sharemap.keys())
for shnum in available_shnums:
if shnum in outstanding_shnums:
# skip ones that are already in transit
continue
if shnum not in self.remaining_sharemap:
# no servers for that shnum. note that DictOfSets removes
# empty sets from the dict for us.
continue
peerid = list(self.remaining_sharemap[shnum])[0]
# get_data will remove that peerid from the sharemap, and add the
# query to self._outstanding_queries
self._status.set_status("Retrieving More Shares")
self.get_data(shnum, peerid)
needed -= 1
if not needed:
break
# at this point, we have as many outstanding queries as we can. If
# needed!=0 then we might not have enough to recover the file.
if needed:
format = ("ran out of peers: "
"have %(have)d shares (k=%(k)d), "
"%(outstanding)d queries in flight, "
"need %(need)d more, "
"found %(bad)d bad shares")
args = {
"have": len(self.shares),
"k": k,
"outstanding": len(self._outstanding_queries),
"need": needed,
"bad": len(self._bad_shares),
}
self.log(format=format, level=log.WEIRD, umid="ezTfjw", **args)
err = NotEnoughSharesError("%s, last failure: %s" %
(format % args, self._last_failure))
if self._bad_shares:
self.log(
"We found some bad shares this pass. You should "
"update the servermap and try again to check "
"more peers",
level=log.WEIRD,
umid="EFkOlA")
err.servermap = self.servermap
raise err
return
def _maybe_send_more_queries(self, k):
# we don't have enough shares yet. Should we send out more queries?
# There are some number of queries outstanding, each for a single
# share. If we can generate 'needed_shares' additional queries, we do
# so. If we can't, then we know this file is a goner, and we raise
# NotEnoughSharesError.
self.log(format=("_maybe_send_more_queries, have=%(have)d, k=%(k)d, "
"outstanding=%(outstanding)d"),
have=len(self.shares), k=k,
outstanding=len(self._outstanding_queries),
level=log.NOISY)
remaining_shares = k - len(self.shares)
needed = remaining_shares - len(self._outstanding_queries)
if not needed:
# we have enough queries in flight already
# TODO: but if they've been in flight for a long time, and we
# have reason to believe that new queries might respond faster
# (i.e. we've seen other queries come back faster, then consider
# sending out new queries. This could help with peers which have
# silently gone away since the servermap was updated, for which
# we're still waiting for the 15-minute TCP disconnect to happen.
self.log("enough queries are in flight, no more are needed",
level=log.NOISY)
return
outstanding_shnums = set([shnum
for (peerid, shnum, started)
in self._outstanding_queries.values()])
# prefer low-numbered shares, they are more likely to be primary
available_shnums = sorted(self.remaining_sharemap.keys())
for shnum in available_shnums:
if shnum in outstanding_shnums:
# skip ones that are already in transit
continue
if shnum not in self.remaining_sharemap:
# no servers for that shnum. note that DictOfSets removes
# empty sets from the dict for us.
continue
peerid = list(self.remaining_sharemap[shnum])[0]
# get_data will remove that peerid from the sharemap, and add the
# query to self._outstanding_queries
self._status.set_status("Retrieving More Shares")
self.get_data(shnum, peerid)
needed -= 1
if not needed:
break
# at this point, we have as many outstanding queries as we can. If
# needed!=0 then we might not have enough to recover the file.
if needed:
format = ("ran out of peers: "
"have %(have)d shares (k=%(k)d), "
"%(outstanding)d queries in flight, "
"need %(need)d more, "
"found %(bad)d bad shares")
args = {"have": len(self.shares),
"k": k,
"outstanding": len(self._outstanding_queries),
"need": needed,
"bad": len(self._bad_shares),
}
self.log(format=format,
level=log.WEIRD, umid="ezTfjw", **args)
err = NotEnoughSharesError("%s, last failure: %s" %
(format % args, self._last_failure))
if self._bad_shares:
self.log("We found some bad shares this pass. You should "
"update the servermap and try again to check "
"more peers",
level=log.WEIRD, umid="EFkOlA")
err.servermap = self.servermap
raise err
return