async def start(self,
node: typing.Optional['Node'] = None,
timeout: typing.Optional[float] = None,
save_now: bool = False):
timeout = timeout or self.config.download_timeout
if self._running.is_set():
return
log.info("start downloader for stream (sd hash: %s)", self.sd_hash)
self._running.set()
try:
await asyncio.wait_for(self.downloader.start(node),
timeout,
loop=self.loop)
except asyncio.TimeoutError:
self._running.clear()
raise DownloadSDTimeout(self.sd_hash)
if self.delayed_stop_task and not self.delayed_stop_task.done():
self.delayed_stop_task.cancel()
self.delayed_stop_task = self.loop.create_task(self._delayed_stop())
if not await self.blob_manager.storage.file_exists(self.sd_hash):
if save_now:
file_name, download_dir = self._file_name, self.download_directory
else:
file_name, download_dir = None, None
self.rowid = await self.blob_manager.storage.save_downloaded_file(
self.stream_hash, file_name, download_dir, 0.0)
if self.status != self.STATUS_RUNNING:
await self.update_status(self.STATUS_RUNNING)
async def _download_stream_from_claim(self, node: 'Node', download_directory: str, claim_info: typing.Dict,
file_name: typing.Optional[str] = None) -> typing.Optional[ManagedStream]:
claim = smart_decode(claim_info['value'])
downloader = StreamDownloader(self.loop, self.config, self.blob_manager, claim.source_hash.decode(),
download_directory, file_name)
try:
downloader.download(node)
await downloader.got_descriptor.wait()
log.info("got descriptor %s for %s", claim.source_hash.decode(), claim_info['name'])
except (asyncio.TimeoutError, asyncio.CancelledError):
log.info("stream timeout")
downloader.stop()
log.info("stopped stream")
raise DownloadSDTimeout(downloader.sd_hash)
rowid = await self._store_stream(downloader)
await self.storage.save_content_claim(
downloader.descriptor.stream_hash, f"{claim_info['txid']}:{claim_info['nout']}"
)
stream = ManagedStream(self.loop, self.blob_manager, rowid, downloader.descriptor, download_directory,
file_name, downloader, ManagedStream.STATUS_RUNNING)
stream.set_claim(claim_info, claim)
self.streams.add(stream)
try:
await stream.downloader.wrote_bytes_event.wait()
self.wait_for_stream_finished(stream)
return stream
except asyncio.CancelledError:
downloader.stop()
log.debug("stopped stream")
await self.stop_stream(stream)
raise DownloadDataTimeout(downloader.sd_hash)
async def load_descriptor(self):
# download or get the sd blob
sd_blob = self.blob_manager.get_blob(self.sd_hash)
if not sd_blob.get_is_verified():
try:
now = self.loop.time()
sd_blob = await asyncio.wait_for(
self.blob_downloader.download_blob(self.sd_hash),
self.config.blob_download_timeout,
loop=self.loop)
log.info("downloaded sd blob %s", self.sd_hash)
self.time_to_descriptor = self.loop.time() - now
except asyncio.TimeoutError:
raise DownloadSDTimeout(self.sd_hash)
# parse the descriptor
self.descriptor = await StreamDescriptor.from_stream_descriptor_blob(
self.loop, self.blob_manager.blob_dir, sd_blob)
log.info("loaded stream manifest %s", self.sd_hash)
async def _download_stream_from_uri(
self,
uri,
timeout: float,
exchange_rate_manager: 'ExchangeRateManager',
file_name: typing.Optional[str] = None) -> ManagedStream:
start_time = self.loop.time()
parsed_uri = parse_lbry_uri(uri)
if parsed_uri.is_channel:
raise ResolveError(
"cannot download a channel claim, specify a /path")
# resolve the claim
resolved = (await self.wallet.ledger.resolve(0, 10, uri)).get(uri, {})
resolved = resolved if 'value' in resolved else resolved.get('claim')
if not resolved:
raise ResolveError(f"Failed to resolve stream at '{uri}'")
if 'error' in resolved:
raise ResolveError(f"error resolving stream: {resolved['error']}")
claim = Claim.from_bytes(binascii.unhexlify(resolved['protobuf']))
outpoint = f"{resolved['txid']}:{resolved['nout']}"
resolved_time = self.loop.time() - start_time
# resume or update an existing stream, if the stream changed download it and delete the old one after
updated_stream, to_replace = await self._check_update_or_replace(
outpoint, resolved['claim_id'], claim)
if updated_stream:
return updated_stream
# check that the fee is payable
fee_amount, fee_address = None, None
if claim.stream.has_fee:
fee_amount = round(
exchange_rate_manager.convert_currency(
claim.stream.fee.currency, "LBC", claim.stream.fee.amount),
5)
max_fee_amount = round(
exchange_rate_manager.convert_currency(
self.config.max_key_fee['currency'], "LBC",
Decimal(self.config.max_key_fee['amount'])), 5)
if fee_amount > max_fee_amount:
msg = f"fee of {fee_amount} exceeds max configured to allow of {max_fee_amount}"
log.warning(msg)
raise KeyFeeAboveMaxAllowed(msg)
balance = await self.wallet.default_account.get_balance()
if lbc_to_dewies(str(fee_amount)) > balance:
msg = f"fee of {fee_amount} exceeds max available balance"
log.warning(msg)
raise InsufficientFundsError(msg)
fee_address = claim.stream.fee.address
# download the stream
download_id = binascii.hexlify(generate_id()).decode()
downloader = StreamDownloader(self.loop, self.config,
self.blob_manager,
claim.stream.source.sd_hash,
self.config.download_dir, file_name)
stream = None
descriptor_time_fut = self.loop.create_future()
start_download_time = self.loop.time()
time_to_descriptor = None
time_to_first_bytes = None
error = None
try:
stream = await asyncio.wait_for(
asyncio.ensure_future(
self.start_downloader(descriptor_time_fut, downloader,
download_id, outpoint, claim,
resolved, file_name)), timeout)
time_to_descriptor = await descriptor_time_fut
time_to_first_bytes = self.loop.time(
) - start_download_time - time_to_descriptor
self.wait_for_stream_finished(stream)
if fee_address and fee_amount and not to_replace:
stream.tx = await self.wallet.send_amount_to_address(
lbc_to_dewies(str(fee_amount)),
fee_address.encode('latin1'))
elif to_replace: # delete old stream now that the replacement has started downloading
await self.delete_stream(to_replace)
except asyncio.TimeoutError:
if descriptor_time_fut.done():
time_to_descriptor = descriptor_time_fut.result()
error = DownloadDataTimeout(downloader.sd_hash)
self.blob_manager.delete_blob(downloader.sd_hash)
await self.storage.delete_stream(downloader.descriptor)
else:
descriptor_time_fut.cancel()
error = DownloadSDTimeout(downloader.sd_hash)
if stream:
await self.stop_stream(stream)
else:
downloader.stop()
if error:
log.warning(error)
if self.analytics_manager:
self.loop.create_task(
self.analytics_manager.send_time_to_first_bytes(
resolved_time,
self.loop.time() - start_time, download_id,
parse_lbry_uri(uri).name, outpoint, None if not stream else
len(stream.downloader.blob_downloader.active_connections),
None if not stream else len(
stream.downloader.blob_downloader.scores),
False if not downloader else downloader.added_fixed_peers,
self.config.fixed_peer_delay
if not downloader else downloader.fixed_peers_delay,
claim.stream.source.sd_hash, time_to_descriptor,
None if not (stream and stream.descriptor) else
stream.descriptor.blobs[0].blob_hash,
None if not (stream and stream.descriptor) else
stream.descriptor.blobs[0].length, time_to_first_bytes,
None if not error else error.__class__.__name__))
if error:
raise error
return stream
async def download_stream_from_uri(
self,
uri,
exchange_rate_manager: 'ExchangeRateManager',
timeout: typing.Optional[float] = None,
file_name: typing.Optional[str] = None,
download_directory: typing.Optional[str] = None,
save_file: bool = True,
resolve_timeout: float = 3.0) -> ManagedStream:
timeout = timeout or self.config.download_timeout
start_time = self.loop.time()
resolved_time = None
stream = None
error = None
outpoint = None
try:
# resolve the claim
parsed_uri = parse_lbry_uri(uri)
if parsed_uri.is_channel:
raise ResolveError(
"cannot download a channel claim, specify a /path")
try:
resolved_result = await asyncio.wait_for(
self.wallet.ledger.resolve(0, 1, uri), resolve_timeout)
except asyncio.TimeoutError:
raise ResolveTimeout(uri)
await self.storage.save_claims_for_resolve([
value for value in resolved_result.values()
if 'error' not in value
])
resolved = resolved_result.get(uri, {})
resolved = resolved if 'value' in resolved else resolved.get(
'claim')
if not resolved:
raise ResolveError(f"Failed to resolve stream at '{uri}'")
if 'error' in resolved:
raise ResolveError(
f"error resolving stream: {resolved['error']}")
claim = Claim.from_bytes(binascii.unhexlify(resolved['protobuf']))
outpoint = f"{resolved['txid']}:{resolved['nout']}"
resolved_time = self.loop.time() - start_time
# resume or update an existing stream, if the stream changed download it and delete the old one after
updated_stream, to_replace = await self._check_update_or_replace(
outpoint, resolved['claim_id'], claim)
if updated_stream:
return updated_stream
content_fee = None
# check that the fee is payable
if not to_replace and claim.stream.has_fee:
fee_amount = round(
exchange_rate_manager.convert_currency(
claim.stream.fee.currency, "LBC",
claim.stream.fee.amount), 5)
max_fee_amount = round(
exchange_rate_manager.convert_currency(
self.config.max_key_fee['currency'], "LBC",
Decimal(self.config.max_key_fee['amount'])), 5)
if fee_amount > max_fee_amount:
msg = f"fee of {fee_amount} exceeds max configured to allow of {max_fee_amount}"
log.warning(msg)
raise KeyFeeAboveMaxAllowed(msg)
balance = await self.wallet.default_account.get_balance()
if lbc_to_dewies(str(fee_amount)) > balance:
msg = f"fee of {fee_amount} exceeds max available balance"
log.warning(msg)
raise InsufficientFundsError(msg)
fee_address = claim.stream.fee.address
content_fee = await self.wallet.send_amount_to_address(
lbc_to_dewies(str(fee_amount)),
fee_address.encode('latin1'))
log.info("paid fee of %s for %s", fee_amount, uri)
download_directory = download_directory or self.config.download_dir
if not file_name and (not self.config.save_files or not save_file):
download_dir, file_name = None, None
stream = ManagedStream(self.loop,
self.config,
self.blob_manager,
claim.stream.source.sd_hash,
download_directory,
file_name,
ManagedStream.STATUS_RUNNING,
content_fee=content_fee,
analytics_manager=self.analytics_manager)
log.info("starting download for %s", uri)
try:
await asyncio.wait_for(stream.setup(
self.node,
save_file=save_file,
file_name=file_name,
download_directory=download_directory),
timeout,
loop=self.loop)
except asyncio.TimeoutError:
if not stream.descriptor:
raise DownloadSDTimeout(stream.sd_hash)
raise DownloadDataTimeout(stream.sd_hash)
if to_replace: # delete old stream now that the replacement has started downloading
await self.delete_stream(to_replace)
stream.set_claim(resolved, claim)
await self.storage.save_content_claim(stream.stream_hash, outpoint)
self.streams[stream.sd_hash] = stream
return stream
except Exception as err:
error = err
if stream and stream.descriptor:
await self.storage.delete_stream(stream.descriptor)
await self.blob_manager.delete_blob(stream.sd_hash)
finally:
if self.analytics_manager and (
error or (stream and
(stream.downloader.time_to_descriptor
or stream.downloader.time_to_first_bytes))):
self.loop.create_task(
self.analytics_manager.send_time_to_first_bytes(
resolved_time,
self.loop.time() - start_time,
None if not stream else stream.download_id, uri,
outpoint, None if not stream else len(
stream.downloader.blob_downloader.
active_connections), None if not stream else
len(stream.downloader.blob_downloader.scores), False
if not stream else stream.downloader.added_fixed_peers,
self.config.fixed_peer_delay
if not stream else stream.downloader.fixed_peers_delay,
None if not stream else stream.sd_hash, None if
not stream else stream.downloader.time_to_descriptor,
None if not (stream and stream.descriptor) else
stream.descriptor.blobs[0].blob_hash,
None if not (stream and stream.descriptor) else
stream.descriptor.blobs[0].length, None if not stream
else stream.downloader.time_to_first_bytes,
None if not error else error.__class__.__name__))
if error:
raise error