async def resolve(self, page, page_size, *uris):
for uri in uris:
try:
parse_lbry_uri(uri)
except URIParseError as err:
return {'error': err.args[0]}
resolutions = await self.network.get_values_for_uris(self.headers.hash().decode(), *uris)
return await self.resolver._handle_resolutions(resolutions, uris, page, page_size)
def _checksig(self, name, value, address):
try:
parse_lbry_uri(name.decode()) # skip invalid names
claim_dict = smart_decode(value)
cert_id = unhexlify(claim_dict.certificate_id)[::-1]
if not self.should_validate_signatures:
return cert_id
if cert_id:
cert_claim = self.db.get_claim_info(cert_id)
if cert_claim:
certificate = smart_decode(cert_claim.value)
claim_dict.validate_signature(address, certificate)
return cert_id
except Exception as e:
pass
def _checksig(self, name, value, address):
try:
parse_lbry_uri(name.decode()) # skip invalid names
cert_id = Claim.FromString(value).publisherSignature.certificateId[::-1] or None
if not self.should_validate_signatures:
return cert_id
if cert_id:
cert_claim = self.db.get_claim_info(cert_id)
if cert_claim:
certificate = smart_decode(cert_claim.value)
claim_dict = smart_decode(value)
claim_dict.validate_signature(address, certificate)
return cert_id
except Exception as e:
pass
async def main():
uris = await get_frontpage_uris()
print("got %i uris" % len(uris))
api = await LBRYAPIClient.get_client()
try:
await api.status()
except (ClientConnectorError, ConnectionError):
await api.session.close()
print("Could not connect to daemon. Are you sure it's running?")
return
first_byte_times = []
for uri in uris:
await api.call(
"file_delete", {
"delete_from_download_dir": True,
"delete_all": True,
"claim_name": parse_lbry_uri(uri).name
})
for i, uri in enumerate(uris):
start = time.time()
try:
await api.call("get", {"uri": uri})
first_byte = time.time()
first_byte_times.append(first_byte - start)
print(f"{i + 1}/{len(uris)} - {first_byte - start} {uri}")
except:
print(
f"{i + 1}/{len(uris)} - timed out in {time.time() - start} {uri}"
)
await api.call(
"file_delete", {
"delete_from_download_dir": True,
"claim_name": parse_lbry_uri(uri).name
})
avg = sum(first_byte_times) / len(first_byte_times)
print()
print(
f"Average time to first byte: {avg} ({len(first_byte_times)} streams)")
print(
f"Started {len(first_byte_times)} Timed out {len(uris) - len(first_byte_times)}"
)
await api.session.close()
async def download_stream_from_uri(self, uri, exchange_rate_manager: 'ExchangeRateManager',
file_name: typing.Optional[str] = None,
timeout: typing.Optional[float] = None) -> typing.Optional[ManagedStream]:
timeout = timeout or self.config.download_timeout
parsed_uri = parse_lbry_uri(uri)
if parsed_uri.is_channel:
raise Exception("cannot download a channel claim, specify a /path")
resolved = (await self.wallet.resolve(uri)).get(uri, {})
resolved = resolved if 'value' in resolved else resolved.get('claim')
if not resolved:
raise ResolveError(
"Failed to resolve stream at lbry://{}".format(uri.replace("lbry://", ""))
)
if 'error' in resolved:
raise ResolveError(f"error resolving stream: {resolved['error']}")
claim = ClaimDict.load_dict(resolved['value'])
fee_amount, fee_address = None, None
if claim.has_fee:
fee_amount = round(exchange_rate_manager.convert_currency(
claim.source_fee.currency, "LBC", claim.source_fee.amount
), 5)
fee_address = claim.source_fee.address.decode()
outpoint = f"{resolved['txid']}:{resolved['nout']}"
existing = self.get_filtered_streams(outpoint=outpoint)
if not existing:
existing.extend(self.get_filtered_streams(sd_hash=claim.source_hash.decode()))
if existing and existing[0].claim_id != resolved['claim_id']:
raise Exception(f"stream for {existing[0].claim_id} collides with existing "
f"download {resolved['claim_id']}")
elif not existing:
existing.extend(self.get_filtered_streams(claim_id=resolved['claim_id']))
if existing and existing[0].sd_hash != claim.source_hash.decode():
log.info("claim contains an update to a stream we have, downloading it")
stream = await self.download_stream_from_claim(
self.node, resolved, file_name, timeout, fee_amount, fee_address, False
)
log.info("started new stream, deleting old one")
await self.delete_stream(existing[0])
return stream
elif existing:
log.info("already have matching stream for %s", uri)
stream = existing[0]
await self.start_stream(stream)
return stream
else:
stream = existing[0]
await self.start_stream(stream)
return stream
log.info("download stream from %s", uri)
return await self.download_stream_from_claim(
self.node, resolved, file_name, timeout, fee_amount, fee_address
)
async def resolve(self, page, page_size, *uris):
try:
for uri in uris:
parsed_uri = parse_lbry_uri(uri)
if parsed_uri.claim_id:
validate_claim_id(parsed_uri.claim_id)
resolutions = await self.network.get_values_for_uris(
self.header_hash, *uris)
return await self._handle_resolutions(resolutions, uris, page,
page_size)
except URIParseError as err:
return {'error': err.args[0]}
except Exception as e:
log.exception(e)
return {'error': str(e)}
async def resolve(self, page, page_size, *uris):
uris = set(uris)
try:
for uri in uris:
parsed_uri = parse_lbry_uri(uri)
if parsed_uri.claim_id:
validate_claim_id(parsed_uri.claim_id)
claim_trie_root = self.ledger.headers.claim_trie_root
resolutions = await self.network.get_values_for_uris(
self.ledger.headers.hash().decode(), *uris)
if len(uris) > 1:
return await self._batch_handle(resolutions, uris, page,
page_size, claim_trie_root)
return await self._handle_resolutions(resolutions, uris, page,
page_size, claim_trie_root)
except URIParseError as err:
return {'error': err.args[0]}
except Exception as e:
log.exception(e)
return {'error': str(e)}
async def _handle_resolve_uri_response(self,
uri,
resolution,
claim_trie_root,
page=0,
page_size=10):
result = {}
parsed_uri = parse_lbry_uri(uri)
certificate_response = None
# parse an included certificate
if 'certificate' in resolution:
certificate_response = resolution['certificate']['result']
certificate_resolution_type = resolution['certificate'][
'resolution_type']
if certificate_resolution_type == "winning" and certificate_response:
if 'height' in certificate_response:
certificate_response = _verify_proof(parsed_uri.name,
claim_trie_root,
certificate_response,
ledger=self.ledger)
elif certificate_resolution_type not in [
'winning', 'claim_id', 'sequence'
]:
raise Exception(
f"unknown response type: {certificate_resolution_type}")
result['certificate'] = await self.parse_and_validate_claim_result(
certificate_response)
result['claims_in_channel'] = len(
resolution.get('unverified_claims_in_channel', []))
# if this was a resolution for a name, parse the result
if 'claim' in resolution:
claim_response = resolution['claim']['result']
claim_resolution_type = resolution['claim']['resolution_type']
if claim_resolution_type == "winning" and claim_response:
if 'height' in claim_response:
claim_response = _verify_proof(parsed_uri.name,
claim_trie_root,
claim_response,
ledger=self.ledger)
elif claim_resolution_type not in [
"sequence", "winning", "claim_id"
]:
raise Exception(
f"unknown response type: {claim_resolution_type}")
result['claim'] = await self.parse_and_validate_claim_result(
claim_response, certificate_response)
# if this was a resolution for a name in a channel make sure there is only one valid
# match
elif 'unverified_claims_for_name' in resolution and 'certificate' in result:
unverified_claims_for_name = resolution[
'unverified_claims_for_name']
channel_info = await self.get_channel_claims_page(
unverified_claims_for_name, result['certificate'], page=1)
claims_in_channel, upper_bound = channel_info
if not claims_in_channel:
log.error("No valid claims for this name for this channel")
elif len(claims_in_channel) > 1:
log.warning("Multiple signed claims for the same name.")
winner = pick_winner_from_channel_path_collision(
claims_in_channel)
if winner:
result['claim'] = winner
else:
log.error("No valid claims for this name for this channel")
else:
result['claim'] = claims_in_channel[0]
# parse and validate claims in a channel iteratively into pages of results
elif 'unverified_claims_in_channel' in resolution and 'certificate' in result:
ids_to_check = resolution['unverified_claims_in_channel']
channel_info = await self.get_channel_claims_page(
ids_to_check,
result['certificate'],
page=page,
page_size=page_size)
claims_in_channel, upper_bound = channel_info
if claims_in_channel:
result['total_claims'] = upper_bound
result['claims_in_channel'] = claims_in_channel
elif 'error' not in result:
return {
'error': 'claim not found',
'success': False,
'uri': str(parsed_uri)
}
# invalid signatures can only return outside a channel
if result.get('claim', {}).get('has_signature', False):
if parsed_uri.path and not result['claim']['signature_is_valid']:
return {
'error': 'claim not found',
'success': False,
'uri': str(parsed_uri)
}
return result
async def _download_stream_from_uri(
self,
uri,
timeout: float,
exchange_rate_manager: 'ExchangeRateManager',
file_name: typing.Optional[str] = None) -> ManagedStream:
start_time = self.loop.time()
parsed_uri = parse_lbry_uri(uri)
if parsed_uri.is_channel:
raise ResolveError(
"cannot download a channel claim, specify a /path")
# resolve the claim
resolved = (await self.wallet.ledger.resolve(0, 10, uri)).get(uri, {})
resolved = resolved if 'value' in resolved else resolved.get('claim')
if not resolved:
raise ResolveError(f"Failed to resolve stream at '{uri}'")
if 'error' in resolved:
raise ResolveError(f"error resolving stream: {resolved['error']}")
claim = Claim.from_bytes(binascii.unhexlify(resolved['protobuf']))
outpoint = f"{resolved['txid']}:{resolved['nout']}"
resolved_time = self.loop.time() - start_time
# resume or update an existing stream, if the stream changed download it and delete the old one after
updated_stream, to_replace = await self._check_update_or_replace(
outpoint, resolved['claim_id'], claim)
if updated_stream:
return updated_stream
# check that the fee is payable
fee_amount, fee_address = None, None
if claim.stream.has_fee:
fee_amount = round(
exchange_rate_manager.convert_currency(
claim.stream.fee.currency, "LBC", claim.stream.fee.amount),
5)
max_fee_amount = round(
exchange_rate_manager.convert_currency(
self.config.max_key_fee['currency'], "LBC",
Decimal(self.config.max_key_fee['amount'])), 5)
if fee_amount > max_fee_amount:
msg = f"fee of {fee_amount} exceeds max configured to allow of {max_fee_amount}"
log.warning(msg)
raise KeyFeeAboveMaxAllowed(msg)
balance = await self.wallet.default_account.get_balance()
if lbc_to_dewies(str(fee_amount)) > balance:
msg = f"fee of {fee_amount} exceeds max available balance"
log.warning(msg)
raise InsufficientFundsError(msg)
fee_address = claim.stream.fee.address
# download the stream
download_id = binascii.hexlify(generate_id()).decode()
downloader = StreamDownloader(self.loop, self.config,
self.blob_manager,
claim.stream.source.sd_hash,
self.config.download_dir, file_name)
stream = None
descriptor_time_fut = self.loop.create_future()
start_download_time = self.loop.time()
time_to_descriptor = None
time_to_first_bytes = None
error = None
try:
stream = await asyncio.wait_for(
asyncio.ensure_future(
self.start_downloader(descriptor_time_fut, downloader,
download_id, outpoint, claim,
resolved, file_name)), timeout)
time_to_descriptor = await descriptor_time_fut
time_to_first_bytes = self.loop.time(
) - start_download_time - time_to_descriptor
self.wait_for_stream_finished(stream)
if fee_address and fee_amount and not to_replace:
stream.tx = await self.wallet.send_amount_to_address(
lbc_to_dewies(str(fee_amount)),
fee_address.encode('latin1'))
elif to_replace: # delete old stream now that the replacement has started downloading
await self.delete_stream(to_replace)
except asyncio.TimeoutError:
if descriptor_time_fut.done():
time_to_descriptor = descriptor_time_fut.result()
error = DownloadDataTimeout(downloader.sd_hash)
self.blob_manager.delete_blob(downloader.sd_hash)
await self.storage.delete_stream(downloader.descriptor)
else:
descriptor_time_fut.cancel()
error = DownloadSDTimeout(downloader.sd_hash)
if stream:
await self.stop_stream(stream)
else:
downloader.stop()
if error:
log.warning(error)
if self.analytics_manager:
self.loop.create_task(
self.analytics_manager.send_time_to_first_bytes(
resolved_time,
self.loop.time() - start_time, download_id,
parse_lbry_uri(uri).name, outpoint, None if not stream else
len(stream.downloader.blob_downloader.active_connections),
None if not stream else len(
stream.downloader.blob_downloader.scores),
False if not downloader else downloader.added_fixed_peers,
self.config.fixed_peer_delay
if not downloader else downloader.fixed_peers_delay,
claim.stream.source.sd_hash, time_to_descriptor,
None if not (stream and stream.descriptor) else
stream.descriptor.blobs[0].blob_hash,
None if not (stream and stream.descriptor) else
stream.descriptor.blobs[0].length, time_to_first_bytes,
None if not error else error.__class__.__name__))
if error:
raise error
return stream
async def claimtrie_getvalueforuri(self,
block_hash,
uri,
known_certificates=None):
# TODO: this thing is huge, refactor
CLAIM_ID = "claim_id"
WINNING = "winning"
SEQUENCE = "sequence"
uri = uri
block_hash = block_hash
try:
parsed_uri = parse_lbry_uri(uri)
except URIParseError as err:
return {'error': err.message}
result = {}
if parsed_uri.is_channel:
certificate = None
# TODO: this is also done on the else, refactor
if parsed_uri.claim_id:
certificate_info = await self.claimtrie_getclaimbyid(
parsed_uri.claim_id)
if certificate_info and certificate_info[
'name'] == parsed_uri.name:
certificate = {
'resolution_type': CLAIM_ID,
'result': certificate_info
}
elif parsed_uri.claim_sequence:
certificate_info = await self.claimtrie_getnthclaimforname(
parsed_uri.name, parsed_uri.claim_sequence)
if certificate_info:
certificate = {
'resolution_type': SEQUENCE,
'result': certificate_info
}
else:
certificate_info = await self.claimtrie_getvalue(
parsed_uri.name, block_hash)
if certificate_info:
certificate = {
'resolution_type': WINNING,
'result': certificate_info
}
if certificate and 'claim_id' not in certificate['result']:
return result
if certificate and not parsed_uri.path:
result['certificate'] = certificate
channel_id = certificate['result']['claim_id']
claims_in_channel = await self.claimtrie_getclaimssignedbyid(
channel_id)
result['unverified_claims_in_channel'] = {
claim['claim_id']: (claim['name'], claim['height'])
for claim in claims_in_channel if claim
}
elif certificate:
result['certificate'] = certificate
channel_id = certificate['result']['claim_id']
claim_ids_matching_name = self.get_signed_claims_with_name_for_channel(
channel_id, parsed_uri.path)
claims = await self.batched_formatted_claims_from_daemon(
claim_ids_matching_name)
claims_in_channel = {
claim['claim_id']: (claim['name'], claim['height'])
for claim in claims
}
result['unverified_claims_for_name'] = claims_in_channel
else:
claim = None
if parsed_uri.claim_id:
claim_info = await self.claimtrie_getclaimbyid(
parsed_uri.claim_id)
if claim_info and claim_info['name'] == parsed_uri.name:
claim = {'resolution_type': CLAIM_ID, 'result': claim_info}
elif parsed_uri.claim_sequence:
claim_info = await self.claimtrie_getnthclaimforname(
parsed_uri.name, parsed_uri.claim_sequence)
if claim_info:
claim = {'resolution_type': SEQUENCE, 'result': claim_info}
else:
claim_info = await self.claimtrie_getvalue(
parsed_uri.name, block_hash)
if claim_info:
claim = {'resolution_type': WINNING, 'result': claim_info}
if (claim and
# is not an unclaimed winning name
(claim['resolution_type'] != WINNING
or proof_has_winning_claim(claim['result']['proof']))):
raw_claim_id = unhexlify(claim['result']['claim_id'])[::-1]
raw_certificate_id = self.db.get_claim_info(
raw_claim_id).cert_id
if raw_certificate_id:
certificate_id = hash_to_hex_str(raw_certificate_id)
certificate = await self.claimtrie_getclaimbyid(
certificate_id)
if certificate:
certificate = {
'resolution_type': CLAIM_ID,
'result': certificate
}
result['certificate'] = certificate
result['claim'] = claim
return result
async def main(uris=None):
if not uris:
uris = await get_frontpage_uris()
conf = Config()
try:
await daemon_rpc(conf, 'status')
except (ClientConnectorError, ConnectionError):
print("Could not connect to daemon")
return 1
print(f"Checking {len(uris)} uris from the front page")
print("**********************************************")
resolvable = []
for name in uris:
resolved = await daemon_rpc(conf, 'resolve', uri=name)
if 'error' not in resolved.get(name, {}):
resolvable.append(name)
print(f"{len(resolvable)}/{len(uris)} are resolvable")
first_byte_times = []
downloaded_times = []
failures = []
download_failures = []
for uri in resolvable:
await daemon_rpc(conf, 'file_delete', delete_from_download_dir=True, claim_name=parse_lbry_uri(uri).name)
for i, uri in enumerate(resolvable):
start = time.time()
try:
await daemon_rpc(conf, 'get', uri)
first_byte = time.time()
first_byte_times.append(first_byte - start)
print(f"{i + 1}/{len(resolvable)} - {first_byte - start} {uri}")
downloaded, amount_downloaded, blobs_in_stream = await wait_for_done(conf, uri)
if downloaded:
downloaded_times.append((time.time() - start) / downloaded)
else:
download_failures.append(uri)
print(f"downloaded {amount_downloaded}/{blobs_in_stream} blobs for {uri} at "
f"{round((blobs_in_stream * (MAX_BLOB_SIZE - 1)) / (time.time() - start) / 1000000, 2)}mb/s\n")
except:
print(f"{i + 1}/{len(uris)} - failed to start {uri}")
failures.append(uri)
return
# await daemon_rpc(conf, 'file_delete', delete_from_download_dir=True, claim_name=parse_lbry_uri(uri).name)
await asyncio.sleep(0.1)
print("**********************************************")
result = f"Tried to start downloading {len(resolvable)} streams from the front page\n" \
f"Worst first byte time: {round(max(first_byte_times), 2)}\n" \
f"Best first byte time: {round(min(first_byte_times), 2)}\n" \
f"95% confidence time-to-first-byte: {confidence(first_byte_times, 1.984)}\n" \
f"99% confidence time-to-first-byte: {confidence(first_byte_times, 2.626)}\n" \
f"Variance: {variance(first_byte_times)}\n" \
f"Started {len(first_byte_times)}/{len(resolvable)} streams"
if failures:
nt = '\n\t'
result += f"\nFailures:\n\t{nt.join([f for f in failures])}"
print(result)
async def claimtrie_getvalueforuri(self,
block_hash,
uri,
known_certificates=None):
# TODO: this thing is huge, refactor
CLAIM_ID = "claim_id"
WINNING = "winning"
SEQUENCE = "sequence"
uri = uri
block_hash = block_hash
try:
parsed_uri = parse_lbry_uri(uri)
except URIParseError as err:
return {'error': err.message}
result = {}
if parsed_uri.contains_channel:
certificate = None
# TODO: this is also done on the else, refactor
if parsed_uri.claim_id:
if len(parsed_uri.claim_id) < CLAIM_ID_MAX_LENGTH:
certificate_info = self.claimtrie_getpartialmatch(
parsed_uri.name, parsed_uri.claim_id)
else:
certificate_info = await self.claimtrie_getclaimbyid(
parsed_uri.claim_id)
if certificate_info and self.claim_matches_name(
certificate_info, parsed_uri.name):
certificate = {
'resolution_type': CLAIM_ID,
'result': certificate_info
}
elif parsed_uri.claim_sequence:
certificate_info = await self.claimtrie_getnthclaimforname(
parsed_uri.name, parsed_uri.claim_sequence)
if certificate_info:
certificate = {
'resolution_type': SEQUENCE,
'result': certificate_info
}
else:
certificate_info = await self.claimtrie_getvalue(
parsed_uri.name, block_hash)
if certificate_info:
certificate = {
'resolution_type': WINNING,
'result': certificate_info
}
if certificate and 'claim_id' not in certificate['result']:
return result
if certificate:
result['certificate'] = certificate
channel_id = certificate['result']['claim_id']
claims_in_channel = self.claimtrie_getclaimssignedbyidminimal(
channel_id)
if not parsed_uri.path:
result['unverified_claims_in_channel'] = {
claim['claim_id']: (claim['name'], claim['height'])
for claim in claims_in_channel
}
else:
# making an assumption that there aren't case conflicts on an existing channel
norm_path = self.normalize_name(parsed_uri.path)
result['unverified_claims_for_name'] = {
claim['claim_id']: (claim['name'], claim['height'])
for claim in claims_in_channel
if self.normalize_name(claim['name']) == norm_path
}
else:
claim = None
if parsed_uri.claim_id:
if len(parsed_uri.claim_id) < CLAIM_ID_MAX_LENGTH:
claim_info = self.claimtrie_getpartialmatch(
parsed_uri.name, parsed_uri.claim_id)
else:
claim_info = await self.claimtrie_getclaimbyid(
parsed_uri.claim_id)
if claim_info and self.claim_matches_name(
claim_info, parsed_uri.name):
claim = {'resolution_type': CLAIM_ID, 'result': claim_info}
elif parsed_uri.claim_sequence:
claim_info = await self.claimtrie_getnthclaimforname(
parsed_uri.name, parsed_uri.claim_sequence)
if claim_info:
claim = {'resolution_type': SEQUENCE, 'result': claim_info}
else:
claim_info = await self.claimtrie_getvalue(
parsed_uri.name, block_hash)
if claim_info:
claim = {'resolution_type': WINNING, 'result': claim_info}
if (claim and
# is not an unclaimed winning name
(claim['resolution_type'] != WINNING
or proof_has_winning_claim(claim['result']['proof']))):
raw_claim_id = unhexlify(claim['result']['claim_id'])[::-1]
raw_certificate_id = self.db.get_claim_info(
raw_claim_id).cert_id
if raw_certificate_id:
certificate_id = hash_to_hex_str(raw_certificate_id)
certificate = await self.claimtrie_getclaimbyid(
certificate_id)
if certificate:
certificate = {
'resolution_type': CLAIM_ID,
'result': certificate
}
result['certificate'] = certificate
result['claim'] = claim
return result
async def main(uris=None, allow_fees=False):
if not uris:
uris = await get_frontpage_uris()
conf = Config()
try:
await daemon_rpc(conf, 'status')
except (ClientConnectorError, ConnectionError):
print("Could not connect to daemon")
return 1
print(f"Checking {len(uris)} uris from the front page")
print("**********************************************")
resolvable = []
for name in uris:
resolved = await daemon_rpc(conf, 'resolve', name)
if 'error' not in resolved.get(name, {}):
if ("fee" not in resolved[name]['claim']['value']['stream']) or allow_fees:
resolvable.append(name)
else:
print(f"{name} has a fee, skipping it")
else:
print(f"failed to resolve {name}: {resolved[name]['error']}")
print(f"attempting to download {len(resolvable)}/{len(uris)} frontpage streams")
first_byte_times = []
download_speeds = []
download_successes = []
failed_to_start = []
download_failures = []
for uri in resolvable:
await daemon_rpc(conf, 'file_delete', delete_from_download_dir=True, claim_name=parse_lbry_uri(uri).name)
for i, uri in enumerate(resolvable):
start = time.time()
try:
await daemon_rpc(conf, 'get', uri)
first_byte = time.time()
first_byte_times.append(first_byte - start)
print(f"{i + 1}/{len(resolvable)} - {first_byte - start} {uri}")
downloaded, amount_downloaded, blobs_in_stream = await wait_for_done(conf, uri)
if downloaded:
download_successes.append(uri)
else:
download_failures.append(uri)
mbs = round((blobs_in_stream * (MAX_BLOB_SIZE - 1)) / (time.time() - start) / 1000000, 2)
download_speeds.append(mbs)
print(f"downloaded {amount_downloaded}/{blobs_in_stream} blobs for {uri} at "
f"{mbs}mb/s")
except:
print(f"{i + 1}/{len(uris)} - failed to start {uri}")
failed_to_start.append(uri)
return
# await daemon_rpc(conf, 'file_delete', delete_from_download_dir=True, claim_name=parse_lbry_uri(uri).name)
await asyncio.sleep(0.1)
print("**********************************************")
result = f"Started {len(first_byte_times)} of {len(resolvable)} attempted front page streams\n" \
f"Worst first byte time: {round(max(first_byte_times), 2)}\n" \
f"Best first byte time: {round(min(first_byte_times), 2)}\n" \
f"95% confidence time-to-first-byte: {confidence(first_byte_times, 1.984)}s\n" \
f"99% confidence time-to-first-byte: {confidence(first_byte_times, 2.626)}s\n" \
f"Variance: {variance(first_byte_times)}\n" \
f"Downloaded {len(download_successes)}/{len(resolvable)}\n" \
f"Best stream download speed: {round(max(download_speeds), 2)}mb/s\n" \
f"Worst stream download speed: {round(min(download_speeds), 2)}mb/s\n" \
f"95% confidence download speed: {confidence(download_speeds, 1.984, False)}mb/s\n" \
f"99% confidence download speed: {confidence(download_speeds, 2.626, False)}mb/s\n"
if failed_to_start:
result += "\nFailed to start:" + "\n".join([f for f in failed_to_start])
if download_failures:
result += "\nFailed to finish:" + "\n".join([f for f in download_failures])
print(result)
webhook = os.environ.get('TTFB_SLACK_TOKEN', None)
if webhook:
await report_to_slack(result, webhook)
async def _download_stream_from_uri(self, uri, exchange_rate_manager: 'ExchangeRateManager',
file_name: typing.Optional[str] = None,
timeout: typing.Optional[float] = None) -> typing.Optional[ManagedStream]:
timeout = timeout or self.config.download_timeout
parsed_uri = parse_lbry_uri(uri)
if parsed_uri.is_channel:
raise ResolveError("cannot download a channel claim, specify a /path")
resolved = (await self.wallet.resolve(uri)).get(uri, {})
resolved = resolved if 'value' in resolved else resolved.get('claim')
if not resolved:
raise ResolveError(
"Failed to resolve stream at lbry://{}".format(uri.replace("lbry://", ""))
)
if 'error' in resolved:
raise ResolveError(f"error resolving stream: {resolved['error']}")
claim = ClaimDict.load_dict(resolved['value'])
fee_amount, fee_address = None, None
if claim.has_fee:
fee_amount = round(exchange_rate_manager.convert_currency(
claim.source_fee.currency, "LBC", claim.source_fee.amount
), 5)
max_fee_amount = round(exchange_rate_manager.convert_currency(
self.config.max_key_fee['currency'], "LBC", self.config.max_key_fee['amount']
), 5)
if fee_amount > max_fee_amount:
msg = f"fee of {fee_amount} exceeds max configured to allow of {max_fee_amount}"
log.warning(msg)
raise KeyFeeAboveMaxAllowed(msg)
balance = await self.wallet.default_account.get_balance()
if lbc_to_dewies(str(fee_amount)) > balance:
msg = f"fee of {fee_amount} exceeds max available balance"
log.warning(msg)
raise InsufficientFundsError(msg)
fee_address = claim.source_fee.address.decode()
outpoint = f"{resolved['txid']}:{resolved['nout']}"
existing = self.get_filtered_streams(outpoint=outpoint)
if not existing:
existing.extend(self.get_filtered_streams(sd_hash=claim.source_hash.decode()))
if existing and existing[0].claim_id != resolved['claim_id']:
raise Exception(f"stream for {existing[0].claim_id} collides with existing "
f"download {resolved['claim_id']}")
existing.extend(self.get_filtered_streams(claim_id=resolved['claim_id']))
if existing and existing[0].sd_hash != claim.source_hash.decode():
log.info("claim contains an update to a stream we have, downloading it")
stream = await self.download_stream_from_claim(
self.node, resolved, file_name, timeout, fee_amount, fee_address, False
)
log.info("started new stream, deleting old one")
if self.analytics_manager:
self.loop.create_task(self.analytics_manager.send_download_started(
stream.download_id, parsed_uri.name, claim.source_hash.decode()
))
await self.delete_stream(existing[0])
return stream
elif existing:
log.info("already have matching stream for %s", uri)
stream = existing[0]
await self.start_stream(stream)
return stream
else:
stream = existing[0]
await self.start_stream(stream)
return stream
log.info("download stream from %s", uri)
stream = await self.download_stream_from_claim(
self.node, resolved, file_name, timeout, fee_amount, fee_address
)
if self.analytics_manager:
self.loop.create_task(self.analytics_manager.send_download_started(
stream.download_id, parsed_uri.name, claim.source_hash.decode()
))
return stream
async def _handle_resolve_uri_response(self,
uri,
resolution,
page=0,
page_size=10,
raw=False):
result = {}
claim_trie_root = self.claim_trie_root
parsed_uri = parse_lbry_uri(uri)
certificate = None
# parse an included certificate
if 'certificate' in resolution:
certificate_response = resolution['certificate']['result']
certificate_resolution_type = resolution['certificate'][
'resolution_type']
if certificate_resolution_type == "winning" and certificate_response:
if 'height' in certificate_response:
height = certificate_response['height']
depth = self.height - height
certificate_result = _verify_proof(
parsed_uri.name,
claim_trie_root,
certificate_response,
height,
depth,
transaction_class=self.transaction_class,
hash160_to_address=self.hash160_to_address)
result[
'certificate'] = await self.parse_and_validate_claim_result(
certificate_result, raw=raw)
elif certificate_resolution_type == "claim_id":
result[
'certificate'] = await self.parse_and_validate_claim_result(
certificate_response, raw=raw)
elif certificate_resolution_type == "sequence":
result[
'certificate'] = await self.parse_and_validate_claim_result(
certificate_response, raw=raw)
else:
log.error("unknown response type: %s",
certificate_resolution_type)
if 'certificate' in result:
certificate = result['certificate']
if 'unverified_claims_in_channel' in resolution:
max_results = len(
resolution['unverified_claims_in_channel'])
result['claims_in_channel'] = max_results
else:
result['claims_in_channel'] = 0
else:
result['error'] = "claim not found"
result['success'] = False
result['uri'] = str(parsed_uri)
else:
certificate = None
# if this was a resolution for a name, parse the result
if 'claim' in resolution:
claim_response = resolution['claim']['result']
claim_resolution_type = resolution['claim']['resolution_type']
if claim_resolution_type == "winning" and claim_response:
if 'height' in claim_response:
height = claim_response['height']
depth = self.height - height
claim_result = _verify_proof(
parsed_uri.name,
claim_trie_root,
claim_response,
height,
depth,
transaction_class=self.transaction_class,
hash160_to_address=self.hash160_to_address)
result[
'claim'] = await self.parse_and_validate_claim_result(
claim_result, certificate, raw)
elif claim_resolution_type == "claim_id":
result['claim'] = await self.parse_and_validate_claim_result(
claim_response, certificate, raw)
elif claim_resolution_type == "sequence":
result['claim'] = await self.parse_and_validate_claim_result(
claim_response, certificate, raw)
else:
log.error("unknown response type: %s", claim_resolution_type)
# if this was a resolution for a name in a channel make sure there is only one valid
# match
elif 'unverified_claims_for_name' in resolution and 'certificate' in result:
unverified_claims_for_name = resolution[
'unverified_claims_for_name']
channel_info = await self.get_channel_claims_page(
unverified_claims_for_name, result['certificate'], page=1)
claims_in_channel, upper_bound = channel_info
if not claims_in_channel:
log.error("No valid claims for this name for this channel")
elif len(claims_in_channel) > 1:
log.warning("Multiple signed claims for the same name.")
winner = pick_winner_from_channel_path_collision(
claims_in_channel)
if winner:
result['claim'] = winner
else:
log.error("No valid claims for this name for this channel")
else:
result['claim'] = claims_in_channel[0]
# parse and validate claims in a channel iteratively into pages of results
elif 'unverified_claims_in_channel' in resolution and 'certificate' in result:
ids_to_check = resolution['unverified_claims_in_channel']
channel_info = await self.get_channel_claims_page(
ids_to_check,
result['certificate'],
page=page,
page_size=page_size)
claims_in_channel, upper_bound = channel_info
if claims_in_channel:
result['claims_in_channel'] = claims_in_channel
elif 'error' not in result:
return {
'error': 'claim not found',
'success': False,
'uri': str(parsed_uri)
}
# invalid signatures can only return outside a channel
if result.get('claim', {}).get('has_signature', False):
if parsed_uri.path and not result['claim']['signature_is_valid']:
return {
'error': 'claim not found',
'success': False,
'uri': str(parsed_uri)
}
return result
async def main(start_daemon=True, uris=None):
if not uris:
uris = await get_frontpage_uris()
conf = Config()
daemon = None
try:
await daemon_rpc(conf, 'status')
except (ClientConnectorError, ConnectionError):
print("Could not connect to daemon")
return 1
print(f"Checking {len(uris)} uris from the front page")
print("**********************************************")
resolvable = []
for name in uris:
resolved = await daemon_rpc(conf, 'resolve', uri=name)
if 'error' not in resolved.get(name, {}):
resolvable.append(name)
print(f"{len(resolvable)}/{len(uris)} are resolvable")
first_byte_times = []
downloaded_times = []
failures = []
download_failures = []
for uri in resolvable:
await daemon_rpc(conf,
'file_delete',
delete_from_download_dir=True,
claim_name=parse_lbry_uri(uri).name)
for i, uri in enumerate(resolvable):
start = time.time()
try:
await daemon_rpc(conf, 'get', uri)
first_byte = time.time()
first_byte_times.append(first_byte - start)
print(f"{i + 1}/{len(resolvable)} - {first_byte - start} {uri}")
# downloaded, msg, blobs_in_stream = await wait_for_done(api, uri)
# if downloaded:
# downloaded_times.append((time.time()-start) / downloaded)
# print(f"{i + 1}/{len(uris)} - downloaded @ {(time.time()-start) / blobs_in_stream}, {msg} {uri}")
# else:
# print(f"failed to downlload {uri}, got {msg}")
# download_failures.append(uri)
except:
print(
f"{i + 1}/{len(uris)} - timeout in {time.time() - start} {uri}"
)
failures.append(uri)
await daemon_rpc(conf,
'file_delete',
delete_from_download_dir=True,
claim_name=parse_lbry_uri(uri).name)
await asyncio.sleep(0.1)
print("**********************************************")
result = f"Tried to start downloading {len(resolvable)} streams from the front page\n" \
f"95% confidence time-to-first-byte: {confidence(first_byte_times, 1.984)}\n" \
f"99% confidence time-to-first-byte: {confidence(first_byte_times, 2.626)}\n" \
f"Variance: {variance(first_byte_times)}\n" \
f"Started {len(first_byte_times)}/{len(resolvable)} streams"
if failures:
nt = '\n\t'
result += f"\nFailures:\n\t{nt.join([f for f in failures])}"
print(result)
if daemon:
await daemon.shutdown()
async def download_stream_from_uri(
self,
uri,
exchange_rate_manager: 'ExchangeRateManager',
timeout: typing.Optional[float] = None,
file_name: typing.Optional[str] = None,
download_directory: typing.Optional[str] = None,
save_file: bool = True,
resolve_timeout: float = 3.0) -> ManagedStream:
timeout = timeout or self.config.download_timeout
start_time = self.loop.time()
resolved_time = None
stream = None
error = None
outpoint = None
try:
# resolve the claim
parsed_uri = parse_lbry_uri(uri)
if parsed_uri.is_channel:
raise ResolveError(
"cannot download a channel claim, specify a /path")
try:
resolved_result = await asyncio.wait_for(
self.wallet.ledger.resolve(0, 1, uri), resolve_timeout)
except asyncio.TimeoutError:
raise ResolveTimeout(uri)
await self.storage.save_claims_for_resolve([
value for value in resolved_result.values()
if 'error' not in value
])
resolved = resolved_result.get(uri, {})
resolved = resolved if 'value' in resolved else resolved.get(
'claim')
if not resolved:
raise ResolveError(f"Failed to resolve stream at '{uri}'")
if 'error' in resolved:
raise ResolveError(
f"error resolving stream: {resolved['error']}")
claim = Claim.from_bytes(binascii.unhexlify(resolved['protobuf']))
outpoint = f"{resolved['txid']}:{resolved['nout']}"
resolved_time = self.loop.time() - start_time
# resume or update an existing stream, if the stream changed download it and delete the old one after
updated_stream, to_replace = await self._check_update_or_replace(
outpoint, resolved['claim_id'], claim)
if updated_stream:
return updated_stream
content_fee = None
# check that the fee is payable
if not to_replace and claim.stream.has_fee:
fee_amount = round(
exchange_rate_manager.convert_currency(
claim.stream.fee.currency, "LBC",
claim.stream.fee.amount), 5)
max_fee_amount = round(
exchange_rate_manager.convert_currency(
self.config.max_key_fee['currency'], "LBC",
Decimal(self.config.max_key_fee['amount'])), 5)
if fee_amount > max_fee_amount:
msg = f"fee of {fee_amount} exceeds max configured to allow of {max_fee_amount}"
log.warning(msg)
raise KeyFeeAboveMaxAllowed(msg)
balance = await self.wallet.default_account.get_balance()
if lbc_to_dewies(str(fee_amount)) > balance:
msg = f"fee of {fee_amount} exceeds max available balance"
log.warning(msg)
raise InsufficientFundsError(msg)
fee_address = claim.stream.fee.address
content_fee = await self.wallet.send_amount_to_address(
lbc_to_dewies(str(fee_amount)),
fee_address.encode('latin1'))
log.info("paid fee of %s for %s", fee_amount, uri)
download_directory = download_directory or self.config.download_dir
if not file_name and (not self.config.save_files or not save_file):
download_dir, file_name = None, None
stream = ManagedStream(self.loop,
self.config,
self.blob_manager,
claim.stream.source.sd_hash,
download_directory,
file_name,
ManagedStream.STATUS_RUNNING,
content_fee=content_fee,
analytics_manager=self.analytics_manager)
log.info("starting download for %s", uri)
try:
await asyncio.wait_for(stream.setup(
self.node,
save_file=save_file,
file_name=file_name,
download_directory=download_directory),
timeout,
loop=self.loop)
except asyncio.TimeoutError:
if not stream.descriptor:
raise DownloadSDTimeout(stream.sd_hash)
raise DownloadDataTimeout(stream.sd_hash)
if to_replace: # delete old stream now that the replacement has started downloading
await self.delete_stream(to_replace)
stream.set_claim(resolved, claim)
await self.storage.save_content_claim(stream.stream_hash, outpoint)
self.streams[stream.sd_hash] = stream
return stream
except Exception as err:
error = err
if stream and stream.descriptor:
await self.storage.delete_stream(stream.descriptor)
await self.blob_manager.delete_blob(stream.sd_hash)
finally:
if self.analytics_manager and (
error or (stream and
(stream.downloader.time_to_descriptor
or stream.downloader.time_to_first_bytes))):
self.loop.create_task(
self.analytics_manager.send_time_to_first_bytes(
resolved_time,
self.loop.time() - start_time,
None if not stream else stream.download_id, uri,
outpoint, None if not stream else len(
stream.downloader.blob_downloader.
active_connections), None if not stream else
len(stream.downloader.blob_downloader.scores), False
if not stream else stream.downloader.added_fixed_peers,
self.config.fixed_peer_delay
if not stream else stream.downloader.fixed_peers_delay,
None if not stream else stream.sd_hash, None if
not stream else stream.downloader.time_to_descriptor,
None if not (stream and stream.descriptor) else
stream.descriptor.blobs[0].blob_hash,
None if not (stream and stream.descriptor) else
stream.descriptor.blobs[0].length, None if not stream
else stream.downloader.time_to_first_bytes,
None if not error else error.__class__.__name__))
if error:
raise error
