async def handle_request(self, request: BlobRequest):
addr = self.transport.get_extra_info('peername')
peer_address, peer_port = addr
responses = []
address_request = request.get_address_request()
if address_request:
responses.append(
BlobPaymentAddressResponse(
lbrycrd_address=self.lbrycrd_address))
availability_request = request.get_availability_request()
if availability_request:
responses.append(
BlobAvailabilityResponse(available_blobs=list(
set(
filter(
lambda blob_hash: blob_hash in self.blob_manager.
completed_blob_hashes,
availability_request.requested_blobs)))))
price_request = request.get_price_request()
if price_request:
responses.append(
BlobPriceResponse(blob_data_payment_rate='RATE_ACCEPTED'))
download_request = request.get_blob_request()
if download_request:
blob = self.blob_manager.get_blob(download_request.requested_blob)
if blob.get_is_verified():
incoming_blob = {
'blob_hash': blob.blob_hash,
'length': blob.length
}
responses.append(
BlobDownloadResponse(incoming_blob=incoming_blob))
self.send_response(responses)
bh = blob.blob_hash[:8]
log.debug("send %s to %s:%i", bh, peer_address, peer_port)
self.started_transfer.set()
try:
sent = await asyncio.wait_for(blob.sendfile(self),
self.transfer_timeout,
loop=self.loop)
self.blob_manager.connection_manager.sent_data(
self.peer_address_and_port, sent)
log.info("sent %s (%i bytes) to %s:%i", bh, sent,
peer_address, peer_port)
except (ConnectionResetError, BrokenPipeError, RuntimeError,
OSError, asyncio.TimeoutError) as err:
if isinstance(err, asyncio.TimeoutError):
log.debug("timed out sending blob %s to %s", bh,
peer_address)
else:
log.debug("stopped sending %s to %s:%i", bh,
peer_address, peer_port)
self.close()
finally:
self.transfer_finished.set()
if responses:
self.send_response(responses)
def data_received(self, data):
request = None
if len(self.buf) + len(data or b'') >= MAX_REQUEST_SIZE:
log.warning("request from %s is too large",
self.peer_address_and_port)
self.close()
return
if data:
self.blob_manager.connection_manager.received_data(
self.peer_address_and_port, len(data))
_, separator, remainder = data.rpartition(b'}')
if not separator:
self.buf += data
return
try:
request = BlobRequest.deserialize(self.buf + data)
self.buf = remainder
except JSONDecodeError:
log.error(
"request from %s is not valid json (%i bytes): %s",
self.peer_address_and_port, len(self.buf + data),
'' if not data else binascii.hexlify(self.buf +
data).decode())
self.close()
return
if not request.requests:
log.error(
"failed to decode request from %s (%i bytes): %s",
self.peer_address_and_port, len(self.buf + data),
'' if not data else binascii.hexlify(self.buf + data).decode())
self.close()
return
self.loop.create_task(self.handle_request(request))
def data_received(self, data):
request = None
if data:
self.blob_manager.connection_manager.received_data(
self.peer_address_and_port, len(data))
message, separator, remainder = data.rpartition(b'}')
if not separator:
self.buf += data
return
try:
request = BlobRequest.deserialize(self.buf + data)
self.buf = remainder
except JSONDecodeError:
addr = self.transport.get_extra_info('peername')
peer_address, peer_port = addr
log.error(
"failed to decode blob request from %s:%i (%i bytes): %s",
peer_address, peer_port, len(data),
'' if not data else binascii.hexlify(data).decode())
if not request:
addr = self.transport.get_extra_info('peername')
peer_address, peer_port = addr
log.warning("failed to decode blob request from %s:%i",
peer_address, peer_port)
self.transport.close()
return
self.loop.create_task(self.handle_request(request))
async def handle_request(self, request: BlobRequest):
addr = self.transport.get_extra_info('peername')
peer_address, peer_port = addr
responses = []
address_request = request.get_address_request()
if address_request:
responses.append(
BlobPaymentAddressResponse(
lbrycrd_address=self.lbrycrd_address))
availability_request = request.get_availability_request()
if availability_request:
responses.append(
BlobAvailabilityResponse(available_blobs=list(
set((filter(
lambda blob_hash: blob_hash in self.blob_manager.
completed_blob_hashes,
availability_request.requested_blobs))))))
price_request = request.get_price_request()
if price_request:
responses.append(
BlobPriceResponse(blob_data_payment_rate='RATE_ACCEPTED'))
download_request = request.get_blob_request()
if download_request:
blob = self.blob_manager.get_blob(download_request.requested_blob)
if blob.get_is_verified():
incoming_blob = {
'blob_hash': blob.blob_hash,
'length': blob.length
}
responses.append(
BlobDownloadResponse(incoming_blob=incoming_blob))
self.send_response(responses)
log.debug("send %s to %s:%i", blob.blob_hash[:8], peer_address,
peer_port)
try:
sent = await blob.sendfile(self)
except (ConnectionResetError, BrokenPipeError, RuntimeError,
OSError):
if self.transport:
self.transport.close()
return
log.info("sent %s (%i bytes) to %s:%i", blob.blob_hash[:8],
sent, peer_address, peer_port)
if responses:
self.send_response(responses)
async def test_server_chunked_request(self):
blob_hash = "7f5ab2def99f0ddd008da71db3a3772135f4002b19b7605840ed1034c8955431bd7079549e65e6b2a3b9c17c773073ed"
server_protocol = BlobServerProtocol(self.loop, self.server_blob_manager, self.server.lbrycrd_address)
transport = asyncio.Transport(extra={'peername': ('ip', 90)})
received_data = BytesIO()
transport.write = received_data.write
server_protocol.connection_made(transport)
blob_request = BlobRequest.make_request_for_blob_hash(blob_hash).serialize()
for byte in blob_request:
server_protocol.data_received(bytes([byte]))
await asyncio.sleep(0.1) # yield execution
self.assertGreater(len(received_data.getvalue()), 0)
async def check_p2p(ip, port):
writer = None
try:
reader, writer = await asyncio.open_connection(ip, port)
writer.write(
BlobRequest.make_request_for_blob_hash('0' * 96).serialize())
return BlobResponse.deserialize(
await
reader.readuntil(b'}')).get_address_response().lbrycrd_address
except OSError:
return None
finally:
if writer:
writer.close()
await writer.wait_closed()
async def _download_blob(self) -> typing.Tuple[int, typing.Optional[asyncio.Transport]]:
"""
:return: download success (bool), keep connection (bool)
"""
request = BlobRequest.make_request_for_blob_hash(self.blob.blob_hash)
blob_hash = self.blob.blob_hash
if not self.peer_address:
addr_info = self.transport.get_extra_info('peername')
self.peer_address, self.peer_port = addr_info
try:
msg = request.serialize()
log.debug("send request to %s:%i -> %s", self.peer_address, self.peer_port, msg.decode())
self.transport.write(msg)
if self.connection_manager:
self.connection_manager.sent_data(f"{self.peer_address}:{self.peer_port}", len(msg))
response: BlobResponse = await asyncio.wait_for(self._response_fut, self.peer_timeout, loop=self.loop)
availability_response = response.get_availability_response()
price_response = response.get_price_response()
blob_response = response.get_blob_response()
if self.closed.is_set():
msg = f"cancelled blob request for {blob_hash} immediately after we got a response"
log.warning(msg)
raise asyncio.CancelledError(msg)
if (not blob_response or blob_response.error) and\
(not availability_response or not availability_response.available_blobs):
log.warning("%s not in availability response from %s:%i", self.blob.blob_hash, self.peer_address,
self.peer_port)
log.warning(response.to_dict())
return self._blob_bytes_received, self.close()
elif availability_response and availability_response.available_blobs and \
availability_response.available_blobs != [self.blob.blob_hash]:
log.warning("blob availability response doesn't match our request from %s:%i",
self.peer_address, self.peer_port)
return self._blob_bytes_received, self.close()
elif not availability_response:
log.warning("response from %s:%i did not include an availability response (we requested %s)",
self.peer_address, self.peer_port, blob_hash)
return self._blob_bytes_received, self.close()
if not price_response or price_response.blob_data_payment_rate != 'RATE_ACCEPTED':
log.warning("data rate rejected by %s:%i", self.peer_address, self.peer_port)
return self._blob_bytes_received, self.close()
if not blob_response or blob_response.error:
log.warning("blob cant be downloaded from %s:%i", self.peer_address, self.peer_port)
return self._blob_bytes_received, self.close()
if not blob_response.error and blob_response.blob_hash != self.blob.blob_hash:
log.warning("incoming blob hash mismatch from %s:%i", self.peer_address, self.peer_port)
return self._blob_bytes_received, self.close()
if self.blob.length is not None and self.blob.length != blob_response.length:
log.warning("incoming blob unexpected length from %s:%i", self.peer_address, self.peer_port)
return self._blob_bytes_received, self.close()
msg = f"downloading {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}," \
f" timeout in {self.peer_timeout}"
log.debug(msg)
msg = f"downloaded {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}"
await asyncio.wait_for(self.writer.finished, self.peer_timeout, loop=self.loop)
log.info(msg)
# await self.blob.finished_writing.wait() not necessary, but a dangerous change. TODO: is it needed?
return self._blob_bytes_received, self.transport
except asyncio.TimeoutError:
return self._blob_bytes_received, self.close()
except (InvalidBlobHashError, InvalidDataError):
log.warning("invalid blob from %s:%i", self.peer_address, self.peer_port)
return self._blob_bytes_received, self.close()
