def _handle_info_response(self, response_dict, peer, request):
if not request.response_identifier in response_dict:
return InvalidResponseError("response identifier not in response")
response = response_dict[request.response_identifier]
if 'error' in response:
if response['error'] == 'RATE_UNSET':
return 0
else:
return InvalidResponseError(
"Got an unknown error from the peer: %s" %
(response['error'], ))
if not 'blob_lengths' in response:
return InvalidResponseError(
"Missing the required field 'blob_lengths'")
raw_blob_lengths = response['blob_lengths']
log.info("Handling %s blob lengths from %s",
str(len(raw_blob_lengths)), str(peer))
log.debug("blobs: %s", str(raw_blob_lengths))
infos = []
unique_hashes = set()
for blob_hash, length in raw_blob_lengths:
if blob_hash in self._valuable_hashes:
peer_score, reference, peer = self._valuable_hashes[blob_hash]
del self._valuable_hashes[blob_hash]
infos.append(
ValuableBlobInfo(blob_hash, length, reference, peer,
peer_score))
unique_hashes.add(blob_hash)
elif blob_hash in request.request_dict['blob_length'][
'blob_hashes']:
unique_hashes.add(blob_hash)
d = self.info_manager.save_blob_infos(infos)
d.addCallback(
lambda _: self.download_manager.add_blobs_to_download(infos))
def pay_or_penalize_peer():
if len(unique_hashes):
self._update_local_score(peer, len(unique_hashes))
peer.update_stats('downloaded_valuable_blob_infos',
len(unique_hashes))
peer.update_score(len(unique_hashes))
else:
self._update_local_score(peer, -.0001)
return len(unique_hashes)
d.addCallback(lambda _: pay_or_penalize_peer())
return d
def _handle_incoming_blob(response_dict, peer, request):
if request.response_identifier not in response_dict:
return InvalidResponseError("response identifier not in response")
if not isinstance(response_dict[request.response_identifier], dict):
return InvalidResponseError(
"response not a dict. got %s" %
type(response_dict[request.response_identifier]))
response = response_dict[request.response_identifier]
if 'error' in response:
# This means we're not getting our blob for some reason
if response['error'] == "RATE_UNSET":
# Stop the download with an error that won't penalize the peer
request.cancel(PriceDisagreementError())
else:
# The peer has done something bad so we should get out of here
return InvalidResponseError(
"Got an unknown error from the peer: %s" %
(response['error'], ))
else:
if 'blob_hash' not in response:
return InvalidResponseError(
"Missing the required field 'blob_hash'")
if not response['blob_hash'] == request.request_dict['requested_blob']:
return InvalidResponseError(
"Incoming blob does not match expected. Incoming: %s. Expected: %s"
% (response['blob_hash'],
request.request_dict['requested_blob']))
if 'length' not in response:
return InvalidResponseError("Missing the required field 'length'")
if not request.blob.set_length(response['length']):
return InvalidResponseError("Could not set the length of the blob")
return True
def _handle_price_response(self, response_dict, peer, request, protocol):
if not request.response_identifier in response_dict:
return InvalidResponseError("response identifier not in response")
assert protocol in self._protocol_prices
response = response_dict[request.response_identifier]
if response == "RATE_ACCEPTED":
return True
else:
del self._protocol_prices[protocol]
self._price_disagreements.append(peer)
return True
def _handle_valuable_blob_response(self, response_dict, peer, request):
if not request.response_identifier in response_dict:
return InvalidResponseError("response identifier not in response")
response = response_dict[request.response_identifier]
if 'error' in response:
if response['error'] == "RATE_UNSET":
return 0
else:
return InvalidResponseError(
"Got an unknown error from the peer: %s" %
(response['error'], ))
if not 'valuable_blob_hashes' in response:
return InvalidResponseError(
"Missing the required field 'valuable_blob_hashes'")
hashes = response['valuable_blob_hashes']
log.info("Handling %s valuable blob hashes from %s", str(len(hashes)),
str(peer))
expire_time = datetime.datetime.now() + datetime.timedelta(minutes=10)
reference = None
unique_hashes = set()
if 'reference' in response:
reference = response['reference']
for blob_hash, peer_score in hashes:
if reference is None:
reference = blob_hash
self._last_blob_hashes_from_peers[peer] = (blob_hash, expire_time)
if not (blob_hash in self._valuable_hashes
or blob_hash in self._blob_infos):
self._valuable_hashes[blob_hash] = (peer_score, reference,
peer)
unique_hashes.add(blob_hash)
if len(unique_hashes):
self._update_local_score(peer, len(unique_hashes))
peer.update_stats('downloaded_valuable_blob_hashes',
len(unique_hashes))
peer.update_score(len(unique_hashes))
else:
self._update_local_score(peer, -.0001)
return len(unique_hashes)
def _handle_availability(self, response_dict, request):
assert request.response_identifier == 'available_blobs'
if 'available_blobs' not in response_dict:
raise InvalidResponseError("response identifier not in response")
log.debug("Received a response to the availability request")
# save available blobs
blob_hashes = response_dict['available_blobs']
for blob_hash in blob_hashes:
if blob_hash in request.request_dict['requested_blobs']:
self.process_available_blob_hash(blob_hash, request)
# everything left in the request is missing
for blob_hash in request.request_dict['requested_blobs']:
self.unavailable_blobs.append(blob_hash)
return True
def _handle_availability(self, response_dict, peer, request):
if not request.response_identifier in response_dict:
raise InvalidResponseError("response identifier not in response")
log.debug("Received a response to the availability request")
blob_hashes = response_dict[request.response_identifier]
for blob_hash in blob_hashes:
if blob_hash in request.request_dict['requested_blobs']:
log.debug(
"The server has indicated it has the following blob available: %s",
blob_hash)
self._available_blobs[peer].append(blob_hash)
if blob_hash in self._unavailable_blobs[peer]:
self._unavailable_blobs[peer].remove(blob_hash)
request.request_dict['requested_blobs'].remove(blob_hash)
for blob_hash in request.request_dict['requested_blobs']:
self._unavailable_blobs[peer].append(blob_hash)
return True
def _handle_availability(self, response_dict, request):
assert request.response_identifier == 'available_blobs'
if 'available_blobs' not in response_dict:
raise InvalidResponseError("response identifier not in response")
log.debug("Received a response to the availability request")
# save available blobs
blob_hashes = response_dict['available_blobs']
if not blob_hashes:
# should not send any more requests as it doesn't have any blob we need
self.update_local_score(-10.0)
return True
for blob_hash in blob_hashes:
if blob_hash in request.request_dict['requested_blobs']:
self.process_available_blob_hash(blob_hash, request)
# everything left in the request is missing
for blob_hash in request.request_dict['requested_blobs']:
self.unavailable_blobs.append(blob_hash)
return True
def _handle_price_response(self, response_dict, request):
assert request.response_identifier == 'blob_data_payment_rate'
if 'blob_data_payment_rate' not in response_dict:
return InvalidResponseError("response identifier not in response")
offer_value = self.protocol_offers.pop(self.protocol)
offer = Offer(offer_value)
offer.handle(response_dict['blob_data_payment_rate'])
self.payment_rate_manager.record_offer_reply(self.peer, offer)
if offer.is_accepted:
log.info("Offered rate %f/mb accepted by %s", offer.rate, self.peer.host)
self.protocol_prices[self.protocol] = offer.rate
return True
elif offer.is_too_low:
log.debug("Offered rate %f/mb rejected by %s", offer.rate, self.peer.host)
return not self.payment_rate_manager.price_limit_reached(self.peer)
else:
log.warning("Price disagreement")
self.requestor._price_disagreements.append(self.peer)
return False
def _handle_price_response(self, response_dict, request):
assert request.response_identifier == 'blob_data_payment_rate'
if 'blob_data_payment_rate' not in response_dict:
return InvalidResponseError("response identifier not in response")
assert self.protocol in self.protocol_prices
rate = self.protocol_prices[self.protocol]
offer = Offer(rate)
offer.handle(response_dict['blob_data_payment_rate'])
self.payment_rate_manager.record_offer_reply(self.peer.host, offer)
if offer.is_accepted:
log.debug("Offered rate %f/mb accepted by %s", rate,
str(self.peer.host))
return True
elif offer.is_too_low:
log.debug("Offered rate %f/mb rejected by %s", rate,
str(self.peer.host))
del self.protocol_prices[self.protocol]
return True
else:
log.warning("Price disagreement")
del self.protocol_prices[self.protocol]
self.requestor._price_disagreements.append(self.peer)
return False
def _handle_discover_response(self, response_dict, peer, request):
if not request.response_identifier in response_dict:
return InvalidResponseError("response identifier not in response")
response = response_dict[request.response_identifier]
blob_infos = []
if 'error' in response:
if response['error'] == 'RATE_UNSET':
return defer.succeed(0)
else:
return InvalidResponseError(
"Got an unknown error from the peer: %s" %
(response['error'], ))
if not 'blob_infos' in response:
return InvalidResponseError(
"Missing the required field 'blob_infos'")
raw_blob_infos = response['blob_infos']
log.info("Handling %s further blobs from %s", str(len(raw_blob_infos)),
str(peer))
log.debug("blobs: %s", str(raw_blob_infos))
for raw_blob_info in raw_blob_infos:
length = raw_blob_info['length']
if length != 0:
blob_hash = raw_blob_info['blob_hash']
else:
blob_hash = None
num = raw_blob_info['blob_num']
revision = raw_blob_info['revision']
iv = raw_blob_info['iv']
signature = raw_blob_info['signature']
blob_info = LiveBlobInfo(blob_hash, num, length, iv, revision,
signature)
log.debug("Learned about a potential blob: %s", str(blob_hash))
if self._verify_blob(blob_info):
if blob_hash is None:
log.info("Setting _final_blob_num to %s", str(num - 1))
self._final_blob_num = num - 1
else:
blob_infos.append(blob_info)
else:
raise ValueError("Peer sent an invalid blob info")
d = self.stream_info_manager.add_blobs_to_stream(
self.stream_hash, blob_infos)
def add_blobs_to_download_manager():
blob_nums = [b.blob_num for b in blob_infos]
log.info(
"Adding the following blob nums to the download manager: %s",
str(blob_nums))
self.download_manager.add_blobs_to_download(blob_infos)
d.addCallback(lambda _: add_blobs_to_download_manager())
def pay_or_penalize_peer():
if len(blob_infos):
self._update_local_score(peer, len(blob_infos))
peer.update_stats('downloaded_crypt_blob_infos',
len(blob_infos))
peer.update_score(len(blob_infos))
else:
self._update_local_score(peer, -.0001)
return len(blob_infos)
d.addCallback(lambda _: pay_or_penalize_peer())
return d